diff --git a/assets/keystone.yaml b/assets/keystone.yaml index 3bc27591..a5326f12 100644 --- a/assets/keystone.yaml +++ b/assets/keystone.yaml @@ -1,28 +1,38 @@ -series: bionic +series: jammy applications: keystone: - charm: cs:keystone + charm: keystone + channel: yoga/stable num_units: 1 options: - openstack-origin: cloud:bionic-rocky worker-multiplier: 0.25 preferred-api-version: 3 - mysql: - charm: cs:percona-cluster - num_units: 1 - options: - innodb-buffer-pool-size: 256M - max-connections: 1000 openstack-dashboard: - charm: cs:openstack-dashboard + charm: openstack-dashboard + channel: yoga/stable num_units: 1 expose: true + mysql: + charm: mysql-innodb-cluster + channel: 8.0/stable + constraints: cores=2 mem=8G root-disk=64G + num_units: 3 options: - openstack-origin: cloud:bionic-rocky + enable-binlogs: true + innodb-buffer-pool-size: 256M + max-connections: 2000 + wait-timeout: 3600 + keystone-mysql-router: + channel: 8.0/stable + charm: mysql-router + openstack-dashboard-mysql-router: + channel: 8.0/stable + charm: mysql-router + relations: -- - keystone:shared-db - - mysql:shared-db -- - openstack-dashboard:identity-service - - keystone:identity-service -- - openstack-dashboard:shared-db - - mysql:shared-db +- [openstack-dashboard:identity-service, keystone:identity-service] +- [keystone-mysql-router:db-router, mysql:db-router] +- [keystone-mysql-router:shared-db, keystone:shared-db] +- [openstack-dashboard-mysql-router:db-router, mysql:db-router] +- [openstack-dashboard-mysql-router:shared-db, openstack-dashboard:shared-db] + diff --git a/generator/k8s_docs_tools/templates/release-notes-header.j2 b/generator/k8s_docs_tools/templates/release-notes-header.j2 index f14084bf..45fd401f 100644 --- a/generator/k8s_docs_tools/templates/release-notes-header.j2 +++ b/generator/k8s_docs_tools/templates/release-notes-header.j2 @@ -1,14 +1,17 @@ --- -wrapper_template: "templates/docs/markdown.html" +wrapper_template: templates/docs/markdown.html markdown_includes: - nav: "kubernetes/docs/shared/_side-navigation.md" + nav: kubernetes/docs/shared/_side-navigation.md context: - title: "{{ release }} Release notes" + title: {{ release }} Release notes description: Release notes for Charmed Kubernetes keywords: kubernetes, release, notes -tags: [news] +tags: + - news sidebar: k8smain-sidebar permalink: {{ release }}/release-notes.html -layout: [base, ubuntu-com] +layout: + - base + - ubuntu-com toc: False --- diff --git a/generator/k8s_docs_tools/templates/supported-versions.j2 b/generator/k8s_docs_tools/templates/supported-versions.j2 index 43e1fcb7..a479fb86 100644 --- a/generator/k8s_docs_tools/templates/supported-versions.j2 +++ b/generator/k8s_docs_tools/templates/supported-versions.j2 @@ -66,11 +66,14 @@ Only the latest three versions of Charmed Kubernetes are supported at any time. ## Professional support -For additional support, learn more about [Ubuntu Pro][support] as well as +For additional support, learn more about [Ubuntu Pro][pro] as well as [managed Kubernetes solutions][managed] from Canonical. +Please visit the Canonical [Support page][support] for more details of our +professional support programmes. +[pro]: /pro [support]: /support [managed]: /kubernetes/managed [releases]: https://github.com/charmed-kubernetes/bundle/tree/main/releases diff --git a/pages/k8s/1.29/release-notes.md b/pages/k8s/1.29/release-notes.md index 3b4aeaf9..7fc81aeb 100644 --- a/pages/k8s/1.29/release-notes.md +++ b/pages/k8s/1.29/release-notes.md @@ -1,22 +1,112 @@ --- -wrapper_template: templates/docs/markdown.html +wrapper_template: "templates/docs/markdown.html" markdown_includes: - nav: kubernetes/docs/shared/_side-navigation.md + nav: "kubernetes/docs/shared/_side-navigation.md" context: - title: 1.29 Release notes + title: "1.29 Release notes" description: Release notes for Charmed Kubernetes keywords: kubernetes, release, notes -tags: - - news +tags: [news] sidebar: k8smain-sidebar permalink: 1.29/release-notes.html -layout: - - base - - ubuntu-com +layout: [base, ubuntu-com] toc: false - --- +# 1.29+ck3 + +### Jun 14, 2024 - `charmed-kubernetes --channel 1.29/stable` + +The release bundle can also be +[downloaded here](https://raw.githubusercontent.com/charmed-kubernetes/bundle/main/releases/1.29/bundle.yaml). + +## Notable Fixes + +### Kubernetes-Control-Plane + +* [LP#2068770](https://bugs.launchpad.net/bugs/2068770) + Upgrade `keystone-credentials` relation with a warning and docs change to [ldap][] +* [LP#2070053](https://bugs.launchpad.net/bugs/2070053) + Upgrade `ceph-client` relation with a warning and docs change to [ceph][] + +# 1.29+ck2 + +### May 30, 2024 - `charmed-kubernetes --channel 1.29/stable` + +The release bundle can also be +[downloaded here](https://raw.githubusercontent.com/charmed-kubernetes/bundle/main/releases/1.29/bundle.yaml). + +## What's new + +### Integration gaps + +- Vault storage: [vault](https://charmhub.io/vault) + - The charm returns support for encryption-at-rest of the secrets in etcd + which were created using a relation to `vault-kv`. The cluster secrets + stored in etcd are encrypted and can only be unlocked by a key which is + stored in Vault. +- Kubernetes-Worker: + - LP#2066049: The charm returns support for the `ingress-proxy` relation. + +## Notable Fixes + +### Kubernetes-Control-Plane + +* [LP#2058269](https://bugs.launchpad.net/bugs/2058269) + Stray "\n" characters after an upgrade to 1.29 + +* [LP#2067427](https://bugs.launchpad.net/bugs/2067427) + Improved build reliability via pinning python dependencies + +### Kubernetes-Worker + +* [LP#2065251](https://bugs.launchpad.net/bugs/2065251) + The charm waits appropriately for tokens when related with cos-agent + +A list of all bug fixes and minor updates in this release can be found at +[the launchpad milestone page for 1.29+ck2](https://launchpad.net/charmed-kubernetes/+milestone/1.29+ck2). + +# 1.29+ck1 Bugfix release + +### April 20, 2024 - `charmed-kubernetes --channel 1.29/stable` + +The release bundle can also be [downloaded here](https://raw.githubusercontent.com/charmed-kubernetes/bundle/main/releases/1.29/bundle.yaml). + +## Notable Fixes + +### Etcd and EasyRSA + +* [LP#2061581](https://bugs.launchpad.net/bugs/2061581) + Could not find a version that satisfies the requirement setuptools>=64 + +### Docker-Registry + +* [LP#2049360](https://bugs.launchpad.net/bugs/2049360) + image corruption with docker-registry charm + +### Kubernetes-Control-Plane + +* [LP#2052140](https://bugs.launchpad.net/bugs/2052140) + grafana agent config not rendered completely + +### Calico-Enterprise + +* [LP#2053143](https://bugs.launchpad.net/bugs/2053143) + Tigera units do not become active after the first installation of the bundle + +### Ceph-CSI + +* [LP#2054486](https://bugs.launchpad.net/bugs/2054486) + ceph-csi charm does not handle ceph-fs correctly + +### Kubernetes-Worker + +* [LP#2054819](https://bugs.launchpad.net/bugs/2054819) + New alert rules shipped from k8s worker + +A list of all bug fixes and minor updates in this release can be found at +[the launchpad milestone page for 1.29+ck1](https://launchpad.net/charmed-kubernetes/+milestone/1.29+ck1). + # 1.29 ### February 12, 2024 - `charmed-kubernetes --channel 1.29/stable` @@ -27,68 +117,74 @@ The release bundle can also be [downloaded here](https://raw.githubusercontent.c ### Charmed Operator Framework (Ops) -We're pleased to announce the completion of the Charmed Kubernetes refactor that began -last year. Core charms have moved from the `reactive` and `pod-spec` styles to the `ops` -framework. This shift aims to enable access to common charm libraries, gain better Juju support, -and provide a more consistent charming experience for community engagement. +We're pleased to announce the completion of the Charmed Kubernetes refactor +that began last year. Core charms have moved from the `reactive` and `pod-spec` +styles to the `ops` framework. This shift aims to enable access to common charm +libraries, gain better Juju support, and provide a more consistent charming +experience for community engagement. ### Out of the box monitoring enhancements -The Canonical Observability Stack (COS) gathers, processes, visualises and alerts on -telemetry signals generated by workloads running both within and outside of Juju. COS -provides an out of the box observability suite relying on the best-in-class open-source -observability tools. +The Canonical Observability Stack (COS) gathers, processes, visualises and +alerts on telemetry signals generated by workloads running both within and +outside of Juju. COS provides an out of the box observability suite relying on +the best-in-class open-source observability tools. This release expands our COS integration so that it includes rich monitoring for the control plane and worker node components of Charmed Kubernetes. ### Ceph CSI -Ceph CSI resource management has been decoupled from the `kubernetes-control-plane` -charm. All new deployments should leverage the [ceph-csi][] charm for Ceph storage -provisioning, including support for CephFS. See the [updated documentation][ceph] for -details on deploying Charmed Kubernetes with Ceph support. +Ceph CSI resource management has been decoupled from the +`kubernetes-control-plane` charm. All new deployments should use the +[ceph-csi][] charm for Ceph storage provisioning, including support for CephFS. +See the [updated documentation][ceph] for details on deploying Charmed +Kubernetes with Ceph support. ### OpenStack integration -OpenStack capabilities (including cinder storage and cloud provider) have been decoupled -from the `kubernetes-control-plane` charm. All new deployments should leverage the new -`openstack-integrator`, `openstack-controller-manager`, and `cinder-csi` charms. See the -[updated documentation][openstack] for more details. +OpenStack capabilities (including cinder storage and cloud provider) have been +decoupled from the `kubernetes-control-plane` charm. All new deployments should +use the new `openstack-integrator`, `openstack-controller-manager`, and +`cinder-csi` charms. See the [updated documentation][openstack] for more +details. ### NVIDIA GPU Operator -The new [nvidia-gpu-operator][] charm simplifies the management of NVIDIA GPU resources -in a Kubernetes cluster. See the [updated documentation][gpu-workers] for details on -deploying Charmed Kubernetes with GPU workers. +The new [nvidia-gpu-operator][] charm simplifies the management of NVIDIA GPU +resources in a Kubernetes cluster. See the [updated documentation][gpu-workers] +for details on deploying Charmed Kubernetes with GPU workers. ### LXD deployment -Updated recommendations for deploying Charmed Kubernetes in a LXD environment are now -available. See the [local install documentation][install-local] for details. +Updated recommendations for deploying Charmed Kubernetes in a LXD environment +are now available. See the [local install documentation][install-local] for +details. ### Manual cloud deployment -Guidelines for deploying Charmed Kubernetes to pre-existing machines are now available. -See the [manual cloud documentation][install-existing] for details. +Guidelines for deploying Charmed Kubernetes to pre-existing machines are now +available. See the [manual cloud documentation][install-existing] for details. ### Container networking enhancements #### Kube-OVN 1.12 -Charmed Kubernetes continues its commitment to advanced container networking with -support for the Kube-OVN CNI. This release includes a Kube-OVN upgrade to v1.12. You can -find more information about features and fixes in the upstream release notes. +Charmed Kubernetes continues its commitment to advanced container networking +with support for the Kube-OVN CNI. This release includes a Kube-OVN upgrade to +v1.12. You can find more information about features and fixes in the upstream +release notes. #### Tigera Calico Enterprise -The `calico-enterprise` charm debuts as a new container networking option for Charmed -Kubernetes in this release. This charm brings advanced Calico networking/network policy -support and is offered as an alternative to the default Calico CNI. +The `calico-enterprise` charm debuts as a new container networking option for +Charmed Kubernetes in this release. This charm brings advanced Calico +networking/network policy support and is offered as an alternative to the +default Calico CNI. ## Fixes -All bug fixes and other feature updates in this release can be found at +All bug fixes and other feature updates in this release can be found at [the launchpad milestone page for 1.29](https://launchpad.net/charmed-kubernetes/+milestone/1.29). @@ -135,8 +231,10 @@ relevant sections of the [upstream release notes][upstream-changelog-1.29]. [rel]: /kubernetes/docs/release-notes [ceph-csi]: https://charmhub.io/ceph-csi?channel=1.29/stable [ceph]: /kubernetes/docs/ceph +[ldap]: /kubernetes/docs/ldap [openstack]: /kubernetes/openstack-integration [nvidia-gpu-operator]: https://charmhub.io/nvidia-gpu-operator?channel=1.29/stable [gpu-workers]: /kubernetes/docs/gpu-workers [install-local]: /kubernetes/docs/install-local [install-existing]: /kubernetes/docs/install-existing +[ldap]: /kuberntes/docs/ldap \ No newline at end of file diff --git a/pages/k8s/1.29/upgrading.md b/pages/k8s/1.29/upgrading.md index 57fb6115..25649f3c 100644 --- a/pages/k8s/1.29/upgrading.md +++ b/pages/k8s/1.29/upgrading.md @@ -18,7 +18,12 @@ toc: False Caution:

This release includes topology changes and new best practices for integrating Charmed Kubernetes with other Juju ecosystem solutions. Be sure to read and understand the *What's new* section of the 1.29 release notes prior to upgrading your cluster.

- Additionally, some features from previous Charmed Kubernetes releases are not yet available in this release. If you rely on a component identified as an *Integration gap* in the Notes and Known Issues section of the release notes, remain on release 1.28 (or earlier) and do not upgrade to 1.29 at this time.

+ Additionally, some features from previous Charmed Kubernetes releases are not yet available in this release. If you rely on a component identified as an *Integration gap* in the Notes and Known Issues section of the release notes, remain on release 1.28 (or earlier) and do not upgrade to 1.29 at this time.
+
+ Some specific scenarios for thoese using particular configurations are also covered in the + Upgrade notes document, particularly concerning those using + observability, LDAP/Keystone integration and Ceph.
+

diff --git a/pages/k8s/ceph.md b/pages/k8s/ceph.md index c27877aa..280b72fb 100644 --- a/pages/k8s/ceph.md +++ b/pages/k8s/ceph.md @@ -13,6 +13,13 @@ layout: [base, ubuntu-com] toc: False --- +
+
+ Note: +

This guide uses the ceph-csi and cephfs operator charms available with Charmed Kubernetes 1.29 and above. For previous versions, see the generic storage guide to integrate Ceph without these charms.

+
+
+ Many workloads that you may want to run on your Kubernetes cluster will require some form of available storage. This guide will help you deploy **Charmed Kubernetes** with **Ceph** container storage support. Available storage backends include `ceph-xfs`, diff --git a/pages/k8s/how-to-cos-lite.md b/pages/k8s/how-to-cos-lite.md index 2f3e8878..cb15966b 100644 --- a/pages/k8s/how-to-cos-lite.md +++ b/pages/k8s/how-to-cos-lite.md @@ -109,12 +109,12 @@ Deploy the grafana-agent: Juju deploy grafana-agent ``` -Relate `grafana-agent` to `k8s`, `kubernetes-control-plane` and `kubernetes-worker`: +Relate `grafana-agent` to charmed kubernetes applications: ``` -juju integrate grafana-agent:cos-agent k8s:cos-agent juju integrate grafana-agent:cos-agent kubernetes-control-plane:cos-agent juju integrate grafana-agent:cos-agent kubernetes-worker:cos-agent +juju integrate grafana-agent:cos-agent kubeapi-load-balancer:cos-agent ``` Relate `grafana-agent` to the COS Lite offered interfaces: diff --git a/pages/k8s/ldap.md b/pages/k8s/ldap.md index 65ebf384..de184080 100644 --- a/pages/k8s/ldap.md +++ b/pages/k8s/ldap.md @@ -24,18 +24,22 @@ or both authentication and authorisation. ## Requirements -* This document assumes you have already [installed][install] **Charmed Kubernetes**. +* This document assumes you have already [installed][install] **Charmed Kubernetes** + * Support for direct LDAP integration via Keystone is dropped beginning in + **Charmed Kubernetes** 1.29, while, upgrades from 1.28 are partially supported. + See [upgrading to 1.29][upgrading] for more detail. * For LDAP authentication, this documentation assumes you already have a suitable LDAP server running. * You will need to install the Keystone client. This can be done by running: ```bash - sudo snap install client-keystone-auth --edge + sudo snap install client-keystone-auth ``` + ## Install Keystone -Note: These instructions assume you are working with the `Queens` release of -**OpenStack**, the default supported version for Ubuntu 18.04 (Bionic) +Note: These instructions assume you are working with the `Yoga` release of +**OpenStack**, the default supported version for Ubuntu 22.04 LTS (Jammy) Keystone should be deployed using **Juju**. This is easily achieved by using a bundle, which will deploy and relate, Keystone, the OpenStack dashboard and a suitable @@ -47,13 +51,6 @@ Deploy the bundle with the following command: juju deploy ./keystone.yaml ``` -You should now add a relation for the kubernetes-control-plane nodes to accept Keystone -credentials: - -```bash -juju integrate keystone:identity-credentials kubernetes-control-plane:keystone-credentials -``` - You can check that the new applications have deployed and are running with: ```bash @@ -81,46 +78,7 @@ juju unexpose openstack-dashboard If you have an existing Keystone application deployed as part of OpenStack in a separate Juju model, it is possible to re-use it for authenticating and authorising users in Kubernetes. -To do so, first deploy the [openstack-integrator charm][openstack-integrator] - -```bash -juju deploy openstack-integrator -``` - -Use 'juju trust' to grant openstack-integrator a permission to access the OpenStack model, -or configure the credentials config parameter manually - -```bash -juju trust openstack-integrator -``` - -Finally add a relation between `kubernetes-control-plane` and `openstack-integrator` - -```bash -juju integrate kubernetes-control-plane:keystone-credentials openstack-integrator:credentials -``` - -## Fetch the Keystone script - -When related to Keystone directly (or to the `openstack-integrator:keystone-credentials` interface), -the Kubernetes master application will generate a utility script. -This should be copied to the local client with: - -```bash -juju scp kubernetes-control-plane/0:kube-keystone.sh ~/kube-keystone.sh -``` - -The file will need to be edited to replace the value for `OS_AUTH_URL`, which should -point at the public address for Keystone, and the username if different. At this point the -file should be sourced: - -```bash -source ~/kube-keystone.sh -``` - -The script should prompt you to enter an additional command to retrieve the token to -login to the OpenStack Dashboard. If this step fails, check that the details in the -`kube-keystone.sh` file are correct. +No extra steps are needed, other than the credentials to access that OpenStack deployment ## Access the OpenStack dashboard @@ -178,50 +136,107 @@ Now ensure the user is added to the project created above. ![dashboard image](https://assets.ubuntu.com/v1/d6149d7c-ldap5.png) +## Deploying the Keystone-Auth Webhook for Kubernetes + +### Understanding the Resources + +Following the upstream docs for [keystone-auth][], the admin should deploy `keystone-auth`. +The following components are key for authentication and authorisation. + +* `Secret/keystone-auth-certs` + * provides the TLS cert/key pair for serving the `keystone-auth` webhook service + * provides the TLS ca cert for contacting keystone (if necessary) +* `ConfigMap/k8s-auth-policy` or `ConfigMap/keystone-sync-policy` + * Configuration for the deployment which translates Keystone users/roles into Kubernetes users/roles +* `Deployment/k8s-keystone-auth` + * defines the PODs backing this service + * defines the image used in the service + * defines the secrets for the service + * defines the configuration for the service + * the `sync-configmap-name` for `keystone-auth`, and `kubernetes-rbac` for authorisation + * the `policy-configmap-name` for `keystone-auth` and Keystone roles +* `ServiceAccount/k8s-keystone`, `ClusterRole/k8s-keystone-auth` and `ClusterRoleBinding/k8s-keystone-auth` + * RBAC rules applied to the deployment to access the cluster `ConfigMap` +* `Service/k8s-keystone-auth-service` + * Service mapping for the above `Deployment/k8s-keystone-auth`. + +### Setting up the Resources + +The following adjustments are required to deploy the service: + +* `Secret/keystone-auth-certs` + * requires the admin to generate a server cert/key pair for the service + * requires the admin to provide the ca cert for the Keystone TLS endpoint (if required) +* `ConfigMap/k8s-auth-policy` (Optional) + * Definitions for mapping keystone user/project/domain/roles to Kubernetes endpoints + * See [keystone-authz-policy][] for details +* `ConfigMap/keystone-sync-policy` (Optional) + * Definitions for mapping keystone user/project/domain/roles to Kubernetes endpoints + * See [keystone-authn-policy][] for details +* `Deployment/k8s-keystone-auth` + * Requires arg `keystone-ca-file` if `keystone-url` is `https` + * Requires arg `policy-configmap-name` or `sync-configmap-name` + * Requires secret volume mapping for the `tls.crt` and `tls.key` + +The following adjustments are required to prepare the API server to use the +authentication endpoint (for both authentication and authorisation) and the +authorisation webhook endpoint. + +* `authn-webhook-endpoint` + **Required** for Authentication and Authorisation + + The API server requires the service endpoint to use as a custom + authentication endpoint. Once applied to the cluster, the + `Service/k8s-keystone-auth-service` should have a `ClusterIP` which will be + used as the `authn-webhook-endpoint`. + + ``` + SVC_IP=$(kubectl get svc -n kube-system k8s-keystone-auth-service -o json | jq -r '.spec.clusterIP') + juju config kubernetes-control-plane authn-webhook-endpoint="https://${SVC_IP}:8443/webhook" + ``` +* `authz-webhook-endpoint` + **Required** only for Authorisation + + The API server requires the service endpoint in the `authorization-webhook-config-file`. + Also, to use this config, the `authorization-mode` must add the `Webhook` mode. + + The crafting of this `webhook-config.yaml` is defined at in the [Keystone examples][keystone-webhook-config] + based on the format defined in the [Kubernetes reference docs][webhook-config] + + First prepare `webhook-config.yaml` using the SVC_IP from above. Then: + ``` + juju config kubernetes-control-plane authorization-webhook-config-file=$(cat webhook-config.yaml) + juju config kubernetes-control-plane authorization-mode="Node,RBAC,Webhook" + ``` + ## Using kubectl with Keystone At this point, Keystone is set up and we have a domain, project, and user -created in Keystone. With the updated config file copied above in -`~/.kube/config`, we can use `kubectl` to authenticate with the api server -via a token from Keystone. The `client-keystone-auth` snap will automate -retrieving a token for us using the environment variables common to -OpenStack such as `OS_USERNAME`. These environment variables are exported in -the `kube-keystone.sh` script we downloaded earlier. To use it, update the -variables in `kube-keystone.sh` to match valid user credentials. Pay -special attention to the `OS_AUTH_URL` variable and ensure it is using an -IP address that is reachable from the client. Source that file into -your environment with `source ./kube-keystone.sh`. Any credentials that -are not supplied via environment variable are queried at run-time for -each invocation of kubectl. - -## Using Keystone with the kubernetes-dashboard - -When using Keystone with Kubernetes, the Kubernetes dashboard is -updated by the charms to use token authentication. This means that a token -from Keystone is required to log in to the Kubernetes dashboard. There is -currently no way to automate this, but the `kube-keystone.sh` file includes -a function called `get_keystone_token`, which uses the `OS_` environment -variables in order to retrieve a token from Keystone. +created in Keystone. + +The authenticating user will need an updated kubeconfig in order to +authenticate with the cluster. One can use `kubectl` to authenticate +with the api server via a token from Keystone. The `client-keystone-auth` +snap automates retrieving a token. + +See the [Client configuration][keystone-client-config] to in order to create +the kubeconfig to use against the Keystone server. + +The client will require the `client-keystone-auth` binary to use this config, +which can be installed using -```bash -source ~/bin/kube-keystone.sh -``` -``` -Function get_keystone_token created. Type get_keystone_token in order to -generate a login token for the Kubernetes dashboard. -``` -Enter the command... -```bash -get_keystone_token -``` -...and a token will be generated: ``` -ccf9b218845f4d67835f8c6a7c2d1cd4 +snap install client-keystone-auth ``` -This token can then be used to log in to the Kubernetes dashboard. +The following variables will need to be set: -![dashboard image](https://assets.ubuntu.com/v1/4b79b35c-token-login.png) +- `OS_USERNAME` +- `OS_PASSWORD` +- `OS_PROJECT_NAME` +- `OS_DOMAIN_NAME` +- `keystone-url` +- `keystone-ca-file` if `keystone-url` is `https` ## LDAP via Keystone @@ -265,7 +280,7 @@ other methods such as RBAC for authorisation but using Keystone for authenticati usernames will come from Keystone, but what they can do in the cluster is controlled by another system. -In order to enable authorization feature in **Charmed Kubernetes** one should change the default config +In order to enable authorisation feature in **Charmed Kubernetes** , change the default config of the charm and switch to **RBAC** authorization mode as follows: ```bash @@ -317,8 +332,13 @@ configuring Keystone/LDAP. [keystone-bundle]: https://raw.githubusercontent.com/juju-solutions/kubernetes-docs/master/assets/keystone.yaml [docs-ldap-keystone]: https://charmhub.io/keystone-ldap [trouble]: /kubernetes/docs/troubleshooting/#troubleshooting-keystoneldap-issues -[openstack-integrator]: /kubernetes/docs/openstack-integration - +[upgrading]: /kubernetes/docs/upgrade-notes +[keystone-auth]: https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/keystone-auth/using-client-keystone-auth.md +[keystone-authz-policy]: https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/keystone-auth/using-keystone-webhook-authenticator-and-authorizer.md#prepare-the-authorization-policy-optional +[keystone-authn-policy]: https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/keystone-auth/using-auth-data-synchronization.md +[keystone-client-config]: https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/keystone-auth/using-keystone-webhook-authenticator-and-authorizer.md#clientkubectl-configuration +[keystone-webhook-config]: https://github.com/kubernetes/cloud-provider-openstack/blob/release-1.30/examples/webhook/keystone-apiserver-webhook.yaml +[webhook-config]: https://kubernetes.io/docs/reference/access-authn-authz/webhook/
diff --git a/pages/k8s/monitoring.md b/pages/k8s/monitoring.md index 64e042f9..65cfcacb 100644 --- a/pages/k8s/monitoring.md +++ b/pages/k8s/monitoring.md @@ -13,6 +13,13 @@ layout: [base, ubuntu-com] toc: False --- +
+
+ Note: +

This page describes enabling an external monitoring stack for Charmed Kubernetes 1.28 and below. For 1.29 and above, we recommend integrating Charmed Kubernetes with the Canonical Observability Stack (COS). See the How-to COS guide for more information.

+
+
+ **Charmed Kubernetes** includes the standard **Kubernetes** dashboard for monitoring your cluster. However, it is often advisable to have a monitoring solution which will run whether the cluster itself is running or not. It diff --git a/pages/k8s/release-notes.md b/pages/k8s/release-notes.md index 710b8acb..032de9d2 100644 --- a/pages/k8s/release-notes.md +++ b/pages/k8s/release-notes.md @@ -15,6 +15,100 @@ toc: False +# 1.29+ck3 + +### Jun 14, 2024 - `charmed-kubernetes --channel 1.29/stable` + +The release bundle can also be +[downloaded here](https://raw.githubusercontent.com/charmed-kubernetes/bundle/main/releases/1.29/bundle.yaml). + +## Notable Fixes + +### Kubernetes-Control-Plane + +* [LP#2068770](https://bugs.launchpad.net/bugs/2068770) + Upgrade `keystone-credentials` relation with a warning and docs change to [ldap][] +* [LP#2070053](https://bugs.launchpad.net/bugs/2070053) + Upgrade `ceph-client` relation with a warning and docs change to [ceph][] + +# 1.29+ck2 + +### May 30, 2024 - `charmed-kubernetes --channel 1.29/stable` + +The release bundle can also be +[downloaded here](https://raw.githubusercontent.com/charmed-kubernetes/bundle/main/releases/1.29/bundle.yaml). + +## What's new + +### Integration gaps + +- Vault storage: [vault](https://charmhub.io/vault) + - The charm returns support for encryption-at-rest of the secrets in etcd + which were created using a relation to `vault-kv`. The cluster secrets + stored in etcd are encrypted and can only be unlocked by a key which is + stored in Vault. +- Kubernetes-Worker: + - LP#2066049: The charm returns support for the `ingress-proxy` relation. + +## Notable Fixes + +### Kubernetes-Control-Plane + +* [LP#2058269](https://bugs.launchpad.net/bugs/2058269) + Stray "\n" characters after an upgrade to 1.29 + +* [LP#2067427](https://bugs.launchpad.net/bugs/2067427) + Improved build reliability via pinning python dependencies + +### Kubernetes-Worker + +* [LP#2065251](https://bugs.launchpad.net/bugs/2065251) + The charm waits appropriately for tokens when related with cos-agent + +A list of all bug fixes and minor updates in this release can be found at +[the launchpad milestone page for 1.29+ck2](https://launchpad.net/charmed-kubernetes/+milestone/1.29+ck2). + +# 1.29+ck1 Bugfix release + +### April 20, 2024 - `charmed-kubernetes --channel 1.29/stable` + +The release bundle can also be [downloaded here](https://raw.githubusercontent.com/charmed-kubernetes/bundle/main/releases/1.29/bundle.yaml). + +## Notable Fixes + +### Etcd and EasyRSA + +* [LP#2061581](https://bugs.launchpad.net/bugs/2061581) + Could not find a version that satisfies the requirement setuptools>=64 + +### Docker-Registry + +* [LP#2049360](https://bugs.launchpad.net/bugs/2049360) + image corruption with docker-registry charm + +### Kubernetes-Control-Plane + +* [LP#2052140](https://bugs.launchpad.net/bugs/2052140) + grafana agent config not rendered completely + +### Calico-Enterprise + +* [LP#2053143](https://bugs.launchpad.net/bugs/2053143) + Tigera units do not become active after the first installation of the bundle + +### Ceph-CSI + +* [LP#2054486](https://bugs.launchpad.net/bugs/2054486) + ceph-csi charm does not handle ceph-fs correctly + +### Kubernetes-Worker + +* [LP#2054819](https://bugs.launchpad.net/bugs/2054819) + New alert rules shipped from k8s worker + +A list of all bug fixes and minor updates in this release can be found at +[the launchpad milestone page for 1.29+ck1](https://launchpad.net/charmed-kubernetes/+milestone/1.29+ck1). + # 1.29 ### February 12, 2024 - `charmed-kubernetes --channel 1.29/stable` @@ -128,11 +222,12 @@ relevant sections of the [upstream release notes][upstream-changelog-1.29]. [rel]: /kubernetes/docs/release-notes [ceph-csi]: https://charmhub.io/ceph-csi?channel=1.29/stable [ceph]: /kubernetes/docs/ceph -[openstack]: /kubernetes/openstack-integration +[openstack]: /kubernetes/docs/openstack-integration [nvidia-gpu-operator]: https://charmhub.io/nvidia-gpu-operator?channel=1.29/stable [gpu-workers]: /kubernetes/docs/gpu-workers [install-local]: /kubernetes/docs/install-local [install-existing]: /kubernetes/docs/install-existing +[ldap]: /kuberntes/docs/ldap diff --git a/pages/k8s/storage.md b/pages/k8s/storage.md index e5ac7573..c9861d8e 100644 --- a/pages/k8s/storage.md +++ b/pages/k8s/storage.md @@ -13,6 +13,13 @@ layout: [base, ubuntu-com] toc: False --- +
+
+ Note: +

For Ceph integration with Charmed Kubernetes 1.29 and above, please see the current Ceph integration guide.

+
+
+ On-disk files in a container are ephemeral and can't be shared with other members of a pod. For some applications, this is not an issue, but for many persistent storage is required. **Charmed Kubernetes** makes it easy to add and configure different types of persistent storage for your **Kubernetes** cluster, as outlined below. For more detail on the concept of storage volumes in **Kubernetes**, please see the [Kubernetes documentation][kubernetes-storage-docs]. diff --git a/pages/k8s/upgrade-notes.md b/pages/k8s/upgrade-notes.md index 4ae8d353..6efadfdf 100644 --- a/pages/k8s/upgrade-notes.md +++ b/pages/k8s/upgrade-notes.md @@ -24,6 +24,202 @@ any of the intervening steps. There is a known issue ([https://bugs.launchpad.net/juju/+bug/1904619](https://bugs.launchpad.net/juju/+bug/1904619)) with container profiles not surviving an upgrade in clouds running on LXD. If your container-based applications fail to work properly after an upgrade, please see this [topic on the troubleshooting page](/kubernetes/docs/troubleshooting#charms-deployed-to-lxd-containers-fail-after-upgradereboot) + + +## Upgrading to 1.29 + +There are several important changes starting in 1.29 that will effect all users: + +- `kubeapi-load-balancer`, `kubernetes-control-plane`, and `kubernetes-worker` charms + can be observed using the COS rather than LMA. +- Dropped specific relations and features which are outsourced to other charms + +### Observability Relations + +These represent relations which were removed in favour of observability with +the Canonical Observability Stack(COS). + +LMA Relations: + +- `nrpe-external-master` (provides: `nrpe-external-master` on KCP and KW) +- `prometheus` (provides: `prometheus-manual` on KCP) +- `scrape` (provides: `prometheus` on KW) +- `grafana` (provides: `grafana-dashboard` ) + +In order to prepare for observability, see the +[Integration with COS Lite docs][cos] which can be performed following an +upgrade of the charms but prior to an upgrade of the Kubernetes cluster. + +### kube-api-endpoint relation dropped + +The `kubernetes-control-plane:kube-api-endpoint` and +`kubernetes-worker:kube-api-endpoint` relations have been removed since these +APIs are are provided by the `kube-control` relation. Ensure these two apps are +linked by `kube-control` relation before removing this relation. + +``` +juju integrate kubernetes-control-plane:kube-control kubernetes-worker:kube-control +juju remove-relation kubernetes-control-plane:kube-api-endpoint kubernetes-worker:kube-api-endpoint +``` + +### loadbalancer relation dropped + +The `kubernetes-control-plane:loadbalancer` relation has been removed in favour +of using the `loadbalancer-internal` and `loadbalancer-external` relations. + +``` +juju integrate kubernetes-control-plane:loadbalancer-internal kubeapi-loadbalancer +juju integrate kubernetes-control-plane:loadbalancer-external kubeapi-loadbalancer +juju remove-relation kubernetes-control-plane:loadbalancer kubeapi-loadbalancer +``` + +### ceph-client relation deprecated + +The `kubernetes-control-plane:ceph-client` relation is being deprecated. + +Ceph integration is still a priority, but continues with the `ceph-csi` charm +which integrates Ceph with Kubernetes. + +After upgrading the `kubernetes-control-plane` charm, the charm +may enter `blocked` status with the message: +`ceph-client relation deprecated, use ceph-csi charm instead`. + +If you see this message, you can resolve it by removing the `ceph-client` +relation: + +``` +juju deploy ceph-csi +juju integrate ceph-csi kubernetes-control-plane +juju integrate ceph-csi ceph-mon +juju remove-relation kubernetes-control-plane:ceph-client ceph-mon +``` + +### Keystone/K8s Authentication management + +Charmed Kubernetes was installing and managing an older version of +keystone-auth which manages authentication and authorisation +through Keystone. + +This service is better suited to be managed externally from the +`kubernetes-control-plane` charm. However, the charm provides the following +upgrade method to maintain the deployment of this service beyond 1.28. + +One can determine if Keystone management is applicable with: + +``` +juju status --relations | grep kubernetes-control-plane:keystone-credentials +``` + +If this is empty, no steps regarding Keystone management are required. + +If this states: + +``` +keystone:identity-credentials kubernetes-control-plane:keystone-credentials keystone-credentials regular +``` + +...then you'll need to prepare a bit before the upgrade. + +#### Resources + +The [upstream Keystone docs][keystone-auth] cover keystone-auth in detail and should be the main reference for implementation details. + +Keystone has two "Auth" options: +1) Authentication of users only called [keystone-authentication][] +2) Authentication and authorisation of users, called [keystone-authorization][] + +Both options require the deployment and management of the [k8s-keystone-auth webhook service][keystone-auth-webhook], +a deployment which provides a service endpoint for the `kubernetes-api-server` to use +as an intermediate to interact with an external Keystone service. + +#### Preparation + +Starting from version 1.29, the `kubernetes-control-plane` charm will drop the following: + +- `kubernetes-control-plane:keystone-credentials` relation +- `keystone-policy` config +- `enable-keystone-authorization` config +- `keystone-ssl-ca` config + +Before upgrading, it is important to capture the state of these config options: + +``` +mkdir keystone-upgrade +juju config kubernetes-control-plane keystone-policy > keystone-upgrade/keystone-policy.yaml +juju config kubernetes-control-plane enable-keystone-authorization > keystone-upgrade/keystone-authorization +juju config kubernetes-control-plane keystone-ssl-ca | base64 -d > keystone-upgrade/keystone-webhook-ca.crt +juju exec -u kubernetes-control-plane/leader -- 'cat /root/cdk/keystone/webhook.yaml' > keystone-upgrade/webhook.yaml +``` + +#### Migration + +After upgrading, the charm will enter a `blocked` state with the status +message: `Keystone credential relation is no longer managed`. This indicates +that the `k8s-keystone-auth` webhook service is still running, but is no longer +managed. + +If `keystone-upgrade/keystone-authorization` contains `true`, then the webhook +should be enabled. This command adds the Keystone authorisation webhook config +and the `Webhook` authorisation mode: + +``` +juju config kubernetes-control-plane \ + authorization-webhook-config-file="$(cat keystone-upgrade/webhook.yaml)" \ + authorization-mode="Node,RBAC,Webhook" +``` + +Finally, acknowledge the charm no longer manages Keystone by removing the relation: + +``` +juju remove-relation kubernetes-control-plane:keystone-credentials keystone +``` + +#### Day 2 Operations + +After migration, the deployment, service, secrets, and policies associated with +`keystone-auth` are no longer handled by the `kubernetes-control-plane` charm. + +The following components remain in the cluster, unmanaged by the charm, and +should be considered managed by the cluster administrators. + +- `Deployment/kube-system/k8s-keystone-auth` +- `Service/kube-system/k8s-keystone-auth-service` +- `Secret/kube-system/keystone-auth-certs` +- `ConfigMap/kube-system/k8s-auth-policy` +- `ClusterRole/k8s-keystone-auth` + + +### Administrative Actions missing + +The `kubernetes-control-plane` and `kubernetes-worker` actions list was +substantially reduced during development of 1.29. The following are no longer +present, but are slated to be reintroduced: + +- `restart` +- `namespace-list` +- `namespace-create` +- `namespace-delete` +- `user-create` +- `user-delete` +- `user-list` +- `apply-manifest` + +### CIS-Benchmark Action missing + +The `kubernetes-control-plane` and `kubernetes-worker` action for cis-benchmark +were removed during the development of the 1.29 charms and an engineering +decision to reintroduce these actions are on-going, but development and testing +incomplete. Details in [LP#2044219][] + +### Automatic labelling of GPU nodes + +While current worker nodes would remain unaffected as they would already be +labelled, the worker charm in 1.29 no longer labels the nodes with `gpu=true` +and `cuda=true`. + +Parity with this feature has been attained by using the [nvidia-gpu-operator][] + + ## Upgrading to 1.24 @@ -382,6 +578,13 @@ You can now proceed with the rest of the upgrade. [dns-provider-config]: https://github.com/juju-solutions/kubernetes/blob/5f4868af82705a0636680a38d7f3ea760d35dadb/cluster/juju/layers/kubernetes-master/config.yaml#L58-L67 [docker-page]: https://jaas.ai/u/containers/docker#configuration [inclusive-naming]: /kubernetes/docs/inclusive-naming +[LP#2044219]: https://bugs.launchpad.net/charm-kubernetes-master/+bug/2044219 +[cos]: kubernetes/docs/how-to-cos-lite +[nvidia-gpu-operator]: https://charmhub.io/nvidia-gpu-operator +[keystone-auth]: https://github.com/kubernetes/cloud-provider-openstack/tree/master/docs/keystone-auth +[keystone-auth-webhook]: https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/keystone-auth/using-keystone-webhook-authenticator-and-authorizer.md#k8s-keystone-auth +[keystone-authentication]: https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/keystone-auth/using-auth-data-synchronization.md#full-example-using-keystone-for-authentication-and-kubernetes-rbac-for-authorization +[keystone-authorization]: https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/keystone-auth/using-keystone-webhook-authenticator-and-authorizer.md#authorization-policy-definitionversion-2