Refine and detail k8s cluster upgrade policy

config/clusters/nasa-esdis/enc-deployer-credentials.secret.json

@@ -1,23 +1,23 @@
 {
 	"AccessKey": {
-		"AccessKeyId": "ENC[AES256_GCM,data:kZTskz6SEOAG7Gs/XJWgb3/rBP8=,iv:0lt2hvWTvim6BQbOHRK1mEM4hOiWaYn1NMk+D6X50uE=,tag:h4kg7OhCjPSkP944bfwl9g==,type:str]",
-		"SecretAccessKey": "ENC[AES256_GCM,data:2reeLNQI/3AoYUvLOt2YMS6rfDif6FKkKy/P1jydTYCOp8WxLJQAWw==,iv:yCFDlAlUMhT++132DYsNj33o1GQeQzf//7812PUeGf0=,tag:chPGkunqefsBCJYktR/4fw==,type:str]",
-		"UserName": "ENC[AES256_GCM,data:UmS7nzeCMNftw5Y3askBVZDCWi3lZug=,iv:JSA+k1aViys9MJiSesJ8Wa1ROjsTp09rTUsuh05D8G0=,tag:Zhc6xFnuH5ZEdyZArr4TAQ==,type:str]"
+		"AccessKeyId": "ENC[AES256_GCM,data:oPKOsurPOC6y9M/1fC+zXWRPlLc=,iv:x/4c7QaVebYIgv1dRz+TrKi26SHx+4Du+6nveh1YCn4=,tag:f98HqtXCHLjwzMqakwJZew==,type:str]",
+		"SecretAccessKey": "ENC[AES256_GCM,data:2wgKlpvDw58nG8I8De1OGuY/iMRBI0YBuOT4PyrS+ftK1z3XaOHE2g==,iv:OjcEoj5TOw0CfgpP0JB3MIUsPX7BuD41UHeKN5X6mWU=,tag:qIPK9FshxAmxQR0eu43obA==,type:str]",
+		"UserName": "ENC[AES256_GCM,data:V3gK4fwvdg6S0Gqm+OMYYQ+CHPaHwx8=,iv:KgqXw4up32lco9+4FJug/59ZxPtp37OSsWeKKuFtM1E=,tag:Dl8I+Q9wZ3iG6vLqtBOuLA==,type:str]"
 	},
 	"sops": {
 		"kms": null,
 		"gcp_kms": [
 			{
 				"resource_id": "projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs",
-				"created_at": "2023-12-06T12:03:15Z",
-				"enc": "CiUA4OM7eCevOe2lP3dSjxmmQYme4j8Z6Y39WZnXn1nIh7yKmXJzEkkAjTWv+vDmpAkn3AgOohJ2cKWKISg+WL5tnT4LSw4+T1f0ErL8Kj7QeGvz5lycB/yGBN00wU8MgkvsoZkSaELOi+CQe0x4pxGi"
+				"created_at": "2024-01-04T23:28:35Z",
+				"enc": "CiUA4OM7eEbZmWcl3SiVG55lrCeNODpLG1k9+umcz3UTw5P2figfEkkAjTWv+piz7M8eFX8o1DVN9zQSgsExgMSC3Ht8Gfy+HQFZMzxSn996JakPNi8m94+LtpZS4cjbqi2jRIEnCMCFhZWNzDRbazvI"
 			}
 		],
 		"azure_kv": null,
 		"hc_vault": null,
 		"age": null,
-		"lastmodified": "2023-12-06T12:03:15Z",
-		"mac": "ENC[AES256_GCM,data:r6AHNmXZIdRXHruqOaC26/1TxUunQE/ZutNdflCqpcKOnQhavhkxWMsITseHtr6FTus/i5HGEqK4TRidCUewSLVKsK7EPEV7jhM9kAL5ERBAUsg26dtS53b4GjOp04joFk3M3UXS/kZpIh+BoxJPnkhPW4rQuaiEL0BYKJbc9ns=,iv:Yg37lNJRrY6z+RN+YlPb2uKhhaXnG0nM+2GqISUwbdo=,tag:gUJEcGXmgKRO0GBKf9rIAA==,type:str]",
+		"lastmodified": "2024-01-04T23:28:36Z",
+		"mac": "ENC[AES256_GCM,data:39hU3JfFxp0+s0adR2OPJv+QXVW81kYXXQ4BzUYAA7cCmRA4o9jQWE+9qAPBVmpWU0zi5WeSAsAJtRV28apcQGcNd4hqtB405dxxsX3cTIlviiAgPggsiS8ZEte5KutCa5BmKkXdMzMxZkMLgbj61K7YlCat82yi3dosHL1rSGA=,iv:jrx0m5+BJab3AeSay9mdZRSwlluDeMiZiVSM60HEUbM=,tag:nK4SFQWnGj+xtB1Jt1YXTw==,type:str]",
 		"pgp": null,
 		"unencrypted_suffix": "_unencrypted",
 		"version": "3.8.1"

config/clusters/nasa-ghg/enc-deployer-credentials.secret.json

@@ -1,23 +1,23 @@
 {
 	"AccessKey": {
-		"AccessKeyId": "ENC[AES256_GCM,data:/SnNrz1NZH14iUvTbwLgPTgPQXM=,iv:PY9j0aRcy92oHYxk5MTc/w0QqbLJnaKfjbk3f3QeHjw=,tag:94iPN87QPhhy+seo45ei4Q==,type:str]",
-		"SecretAccessKey": "ENC[AES256_GCM,data:7Ve2qkiVgs2oTRZ4DmicdAYcTtkv1wasASlCLgTqQYTRrmrnRsM3IA==,iv:ZAGR/Oz8oKNczgnt6JNw9FtrN4EyiNazZQBva4fQ9zQ=,tag:6NeNP7Yoq5fqAbCWGVpK9Q==,type:str]",
-		"UserName": "ENC[AES256_GCM,data:Z90ncTWifCB0NjDp28vWKoWNTwaOT2g=,iv:TEG1Z1Asy1ejwKWX2ylqf0CoCYn9sMoy0D+ULnBiYfo=,tag:SZWVgDCM1jJozZsUClNBYQ==,type:str]"
+		"AccessKeyId": "ENC[AES256_GCM,data:vXMsN91BXqgdF7KXxqX86p61aDo=,iv:P+RyTSlHiqSB9wu0lWJztJkuHwcPsmGQXNEzHM65XR4=,tag:Q/4fuVHgLFxXRNQ0LxCcvg==,type:str]",
+		"SecretAccessKey": "ENC[AES256_GCM,data:px56aI9GMUgkvcbYj001XHhxfMo1nWaWxiBuuIT1x1+N70fE8YFaIw==,iv:xmhfzt1JqlEHESWZnirojcVBvsJTk9hqF7Om3YKUHOs=,tag:y3PN+YRl50FDieRIl9QRyA==,type:str]",
+		"UserName": "ENC[AES256_GCM,data:pQeaY7atRNpI/i8ah9AKprHp194sbuA=,iv:74ae090GbgcxudUi/cDeV46CJX9v5qNBKK3+zk0HCu0=,tag:dT7+f0JuJC1NfaizUFbB9w==,type:str]"
 	},
 	"sops": {
 		"kms": null,
 		"gcp_kms": [
 			{
 				"resource_id": "projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs",
-				"created_at": "2023-12-12T11:39:58Z",
-				"enc": "CiUA4OM7ePvn5wSbGI+YBkf8tpE5da9gpXo7uUg76CB/PH9qw4XgEkkAjTWv+kuW9BPeM6kaAzOlTvAtq6ln/ozWmCZ8oBCvSzC3GzzyNrBPiYtL0RdglmxCVeuVuJbmYQ5f24/vlyKJ7z521fXw1lY6"
+				"created_at": "2024-01-04T23:10:31Z",
+				"enc": "CiUA4OM7eKIg03EAlCM64KT1ZKrqMBL0QvL7zgFEAVQiIWcgjSWeEkkAjTWv+pfKWeZpLUe4MO5M3t7qdwzWD9mP7QpARXmCvwULPVP79EKzCHaErQHIa0Sq8g3ZvZHn2IpaSEGqvSg08rnL2UvPXWn2"
 			}
 		],
 		"azure_kv": null,
 		"hc_vault": null,
 		"age": null,
-		"lastmodified": "2023-12-12T11:39:58Z",
-		"mac": "ENC[AES256_GCM,data:FlX4TV328tvYkRDq0IN4lsspRtk6dtmyTDBjHiWReNS/JmPhtpreAcbkTrMgbj9aSPlJ44hEiPJcTPvbnEBqfouYkgWThWHI4uGwjLMj/EW82HhTDmSR5QNqaHXDyVKjwYfLBuHpGVcqh5dglMoWcAa6lcbcyln+DHr3a3Fa0m0=,iv:tc/d5UgJcdN12+Pvip9BEnwuM1ENqHp3gM2+poPOv2Y=,tag:PFokY3k7piFPYObrPzOuiQ==,type:str]",
+		"lastmodified": "2024-01-04T23:10:31Z",
+		"mac": "ENC[AES256_GCM,data:ThsEIM/YzZyCVCUsnSqoR3gMC+WDt6PyFHY4bSdI5zwD7gbW3V//GK/xUz0XcYsTmKqgL3b99mv3yK1EOyDlYj0LZoLOLZg8nh7Cc9JaiFA4fuvDu/b5r2hA1zYLERxJnJftlqG5sA4nDzv1wnNWf/LL0jzxApvlHty7g1Oa5BU=,iv:sp6AifQVyXgrHTVrIPEGzekhvyfA5syIA2YthhMuJsQ=,tag:k7L2uvgirZkoV5tG1aKLkA==,type:str]",
 		"pgp": null,
 		"unencrypted_suffix": "_unencrypted",
 		"version": "3.8.1"

config/clusters/nasa-veda/enc-deployer-credentials.secret.json

@@ -1,23 +1,23 @@
 {
 	"AccessKey": {
-		"AccessKeyId": "ENC[AES256_GCM,data:snwBYJoxlHiqiy1Z5P+Te305S+o=,iv:mw8v47wbEFYFInVALmqai708PbcGilD9a2EooiqPj6I=,tag:wd9JeH7e6dsArPf2HFckvQ==,type:str]",
-		"SecretAccessKey": "ENC[AES256_GCM,data:Nm7cPP0TVZbXrB8dI1eBB5DdQndYsVjZwC6nzoMyehFLuH8MxA3A9w==,iv:nYWAH7JH9AGX7F7PRjp2uHvFM3GOHi4gx9Ofzdc8VEE=,tag:VwJK6wOFAZpvuTdf1cZe+g==,type:str]",
-		"UserName": "ENC[AES256_GCM,data:0wO9NjNEYDI7VEwbaABNuflsbKHRcCI=,iv:UaQaXwPjgsnHObJWunkSrhxCb5jnlho4m1roMJuH/SY=,tag:pRz4GNk89O13KK0iL6J/kg==,type:str]"
+		"AccessKeyId": "ENC[AES256_GCM,data:3raNFoP4EOz5brXR10QdppGdRoQ=,iv:CHhvKsS+eZwCA+ryRkiHKGuyzH5YI1fCsChalBdm/3M=,tag:j92XTrA1OxZBQGZIj4eQSg==,type:str]",
+		"SecretAccessKey": "ENC[AES256_GCM,data:uJfSLuUpvtxEPQLV7AUY5+g948dr0kyMGcfSIqRDaCI1bGKOZjjx6g==,iv:lnlnK8ZcvAswaiE5U4qPE6LCTIubgPfWEoq9zIvvaRQ=,tag:0IwOhy99k21ifv0jzMwhMQ==,type:str]",
+		"UserName": "ENC[AES256_GCM,data:uCxkY2QPKebMKHAnGRi0/LZg8vePcMI=,iv:+E1dTSDk93Gtsd9pPVoyFULI641SBkjFOaXeG5Z1J+o=,tag:gpNsfqEsqqGKLY1ytqMihw==,type:str]"
 	},
 	"sops": {
 		"kms": null,
 		"gcp_kms": [
 			{
 				"resource_id": "projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs",
-				"created_at": "2023-12-06T16:20:17Z",
-				"enc": "CiUA4OM7eGFJsgOtkHhpuwtBk7FVibsSQb/81TJhM3geXJ4yP0BZEkkAjTWv+tD1IGTInNdjJdb3sll1n9vPIEJ+EWnp53nRYe3+WBOV53BPZMOq1usbIBDMVkriNHw8YtIRO2lfNVBFKhKaUd5TCgPH"
+				"created_at": "2024-01-04T23:06:56Z",
+				"enc": "CiUA4OM7eM8HZ6FS+hnOqcl86XpGiYj9GeOW3aeoFnMpu6VkVoDsEkkAjTWv+iU0Yu+kXQ8FodYPhsM8asnqX1CjZ5pqIQCU0fY1qKxZ/SQNwL4rPb44VIWeYtnne3M6pZJOwtDQvtElowmuOkjEfrsz"
 			}
 		],
 		"azure_kv": null,
 		"hc_vault": null,
 		"age": null,
-		"lastmodified": "2023-12-06T16:20:17Z",
-		"mac": "ENC[AES256_GCM,data:igdFyuAFjSfqiQVVIhQap23ODCDRe5azpaKKyGL5G1dQ34voJfUhAq0KR2bAlfHBGSAo6l9qI18vcdzSjHmkPsxu1sJWhN3H3l0C6HubIWbplXmlv0t7b8PuBASx0DSy9LxsIVPygXP9/88GyU+nP2ywwdgrVZ8fxxqS1YuYkDc=,iv:KmpCe1L4io1sDEWCCyqGS69bz7cNM2+/8DeJuVYyaeU=,tag:qF+TP2UjKp25GNBw7cQAhQ==,type:str]",
+		"lastmodified": "2024-01-04T23:06:57Z",
+		"mac": "ENC[AES256_GCM,data:yU4oEi/fef0HKtezbjkjcNmKM/1LNUhdUPQMOHixsstaGoETva6BHYh+5YtHAdepDPfEd76y9vC4OeIZ2Aenilq6Qe3B2ar+lIGy0CA/Ph1KjS1+Q9/5v0vc9hlReNtpjoqz3xtx320sP0NwlIlKXNwg52xQQzQrvGOvFjzhLoA=,iv:GBR/zD7oynCaDSvF+Ec937VHKgNCcNIuuTO7EtKcfRg=,tag:EdbJ9VOE8/0TCA35SsMyKQ==,type:str]",
 		"pgp": null,
 		"unencrypted_suffix": "_unencrypted",
 		"version": "3.8.1"

docs/howto/upgrade-cluster/index.md

@@ -11,19 +11,26 @@ clusters on AWS.
 
 ## Upgrade policy
 
-We aim to ensure we use a k8s version for the control plane and node groups that is at least **five minor versions** behind the latest one available at any given time.
-
-Ideally, the following rules should also be respected:
-
-1. Every new cluster we deploy should be using the latest available kubernetes version.
-1. All of the clusters deployed in a cloud provider should be using the same version.
-1. Check if new upgrades are needed at least every 3 months.
-
-
-```{warning}
-As of now, we have not yet established practices on how to ensure these upgrades happen according to the policy above. Establishing this is tracked in [this GitHub
-issue](https://github.com/2i2c-org/infrastructure/issues/412).
-```
+1. To keep our k8s cluster's control plane and node pools upgraded to the latest
+   _three_ and _four_ [official minor k8s versions] respectively at all times.
+2. To await a level of maturity for minor k8s versions before we adopt them.
+
+   | Kubernetes distribution | Our maturity criteria                     |
+   | -                       | -                                         |
+   | GKE                     | Part of [GKE's regular release channel]   |
+   | EKS                     | [Supported by `eksctl`] and is GKE mature |
+   | AKS                     | Listed as [generally available on AKS]    |
+3. To upgrade k8s cluster's control plane and node pools at least _twice_ and
+   _once_ per year respectively.
+4. To not disrupt user nodes with running users, by instead rolling out new user
+   node pools if needed and cleaning up the old at a later time.
+5. To check if actions needs to be scheduled related to this in the beginning of
+   every quarter.
+
+[official minor k8s versions]: https://kubernetes.io/releases/
+[gke's regular release channel]: https://cloud.google.com/kubernetes-engine/docs/release-notes-regular
+[supported by `eksctl`]: https://eksctl.io/getting-started/#basic-cluster-creation
+[generally available on aks]: https://learn.microsoft.com/en-gb/azure/aks/supported-kubernetes-versions?tabs=azure-cli#aks-kubernetes-release-calendar
 
 ```{toctree}
 :maxdepth: 1