diff --git a/argocd-helm-charts/cluster-autoscaler/Chart.lock b/argocd-helm-charts/cluster-autoscaler/Chart.lock index 5a89aeffc..fbc15c63e 100644 --- a/argocd-helm-charts/cluster-autoscaler/Chart.lock +++ b/argocd-helm-charts/cluster-autoscaler/Chart.lock @@ -1,6 +1,6 @@ dependencies: - name: cluster-autoscaler repository: https://kubernetes.github.io/autoscaler - version: 9.23.0 -digest: sha256:fa7f4ed08b3760a092925a6f2331815d1c077ffa4008b1c647803c23d199e265 -generated: "2023-02-03T05:05:12.789843478Z" + version: 9.29.3 +digest: sha256:697a1032e0023d622e30794e8482b9027764b5654f89e76792d2a129f9a61168 +generated: "2023-10-09T13:43:51.250355135+05:30" diff --git a/argocd-helm-charts/cluster-autoscaler/Chart.yaml b/argocd-helm-charts/cluster-autoscaler/Chart.yaml index e873f4c98..38c716ec6 100644 --- a/argocd-helm-charts/cluster-autoscaler/Chart.yaml +++ b/argocd-helm-charts/cluster-autoscaler/Chart.yaml @@ -3,6 +3,6 @@ name: cluster-autoscaler version: 1.21.1 dependencies: - name: cluster-autoscaler - version: 9.23.0 + version: 9.29.3 repository: https://kubernetes.github.io/autoscaler #repository: "oci://ghcr.io/Obmondo" diff --git a/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/Chart.yaml b/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/Chart.yaml index 0407ca0c6..f49fa4929 100644 --- a/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/Chart.yaml +++ b/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 1.23.0 +appVersion: 1.27.2 description: Scales Kubernetes worker nodes within autoscaling groups. home: https://github.com/kubernetes/autoscaler icon: https://github.com/kubernetes/kubernetes/raw/master/logo/logo.png @@ -10,4 +10,4 @@ name: cluster-autoscaler sources: - https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler type: application -version: 9.23.0 +version: 9.29.3 diff --git a/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/README.md b/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/README.md index 578e570c8..dd6171b78 100644 --- a/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/README.md +++ b/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/README.md @@ -2,20 +2,20 @@ Scales Kubernetes worker nodes within autoscaling groups. -## TL;DR: +## TL;DR ```console $ helm repo add autoscaler https://kubernetes.github.io/autoscaler # Method 1 - Using Autodiscovery $ helm install my-release autoscaler/cluster-autoscaler \ ---set 'autoDiscovery.clusterName'= + --set 'autoDiscovery.clusterName'= # Method 2 - Specifying groups manually $ helm install my-release autoscaler/cluster-autoscaler \ ---set "autoscalingGroups[0].name=your-asg-name" \ ---set "autoscalingGroups[0].maxSize=10" \ ---set "autoscalingGroups[0].minSize=1" + --set "autoscalingGroups[0].name=your-asg-name" \ + --set "autoscalingGroups[0].maxSize=10" \ + --set "autoscalingGroups[0].minSize=1" ``` ## Introduction @@ -68,10 +68,10 @@ Either: To create a valid configuration, follow instructions for your cloud provider: -* [AWS](#aws---using-auto-discovery-of-tagged-instance-groups) -* [GCE](#gce) -* [Azure AKS](#azure-aks) -* [OpenStack Magnum](#openstack-magnum) +- [AWS](#aws---using-auto-discovery-of-tagged-instance-groups) +- [GCE](#gce) +- [Azure AKS](#azure-aks) +- [OpenStack Magnum](#openstack-magnum) ### AWS - Using auto-discovery of tagged instance groups @@ -84,13 +84,19 @@ Auto-discovery finds ASGs tags as below and automatically manages them based on - Set (option) `awsAccessKeyID=` and `awsSecretAccessKey=` if you want to [use AWS credentials directly instead of an instance role](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#using-aws-credentials) ```console -$ helm install my-release autoscaler/cluster-autoscaler --set autoDiscovery.clusterName= --set awsRegion= +$ helm install my-release autoscaler/cluster-autoscaler \ + --set autoDiscovery.clusterName= \ + --set awsRegion= ``` Alternatively with your own AWS credentials ```console -$ helm install my-release autoscaler/cluster-autoscaler --set autoDiscovery.clusterName= --set awsRegion= --set awsAccessKeyID= --set awsSecretAccessKey= +$ helm install my-release autoscaler/cluster-autoscaler \ + --set autoDiscovery.clusterName= \ + --set awsRegion= \ + --set awsAccessKeyID= \ + --set awsSecretAccessKey= ``` #### Specifying groups manually @@ -102,15 +108,14 @@ Without autodiscovery, specify an array of elements each containing ASG name, mi ```console $ helm install my-release autoscaler/cluster-autoscaler \ ---set "autoscalingGroups[0].name=your-asg-name" \ ---set "autoscalingGroups[0].maxSize=10" \ ---set "autoscalingGroups[0].minSize=1" + --set "autoscalingGroups[0].name=your-asg-name" \ + --set "autoscalingGroups[0].maxSize=10" \ + --set "autoscalingGroups[0].minSize=1" ``` #### Auto-discovery -For auto-discovery of instances to work, they must be tagged with the keys in `.Values.autoDiscovery.tags`, which by default are -`k8s.io/cluster-autoscaler/enabled` and `k8s.io/cluster-autoscaler/` +For auto-discovery of instances to work, they must be tagged with the keys in `.Values.autoDiscovery.tags`, which by default are `k8s.io/cluster-autoscaler/enabled` and `k8s.io/cluster-autoscaler/`. The value of the tag does not matter, only the key. @@ -147,7 +152,7 @@ spec: In this example you would need to `--set autoDiscovery.clusterName=my.cluster.internal` when installing. -It is not recommended to try to mix this with setting `autoscalingGroups` +It is not recommended to try to mix this with setting `autoscalingGroups`. See [autoscaler AWS documentation](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#auto-discovery-setup) for a more discussion of the setup. @@ -163,9 +168,9 @@ To use Managed Instance Group (MIG) auto-discovery, provide a YAML file setting ```console $ helm install my-release autoscaler/cluster-autoscaler \ ---set "autoscalingGroupsnamePrefix[0].name=your-ig-prefix,autoscalingGroupsnamePrefix[0].maxSize=10,autoscalingGroupsnamePrefi[0].minSize=1" \ ---set autoDiscovery.clusterName= \ ---set cloudProvider=gce + --set "autoscalingGroupsnamePrefix[0].name=your-ig-prefix,autoscalingGroupsnamePrefix[0].maxSize=10,autoscalingGroupsnamePrefi[0].minSize=1" \ + --set autoDiscovery.clusterName= \ + --set cloudProvider=gce ``` Note that `your-ig-prefix` should be a _prefix_ matching one or more MIGs, and _not_ the full name of the MIG. For example, to match multiple instance groups - `k8s-node-group-a-standard`, `k8s-node-group-b-gpu`, you would use a prefix of `k8s-node-group-`. @@ -174,7 +179,7 @@ In the event you want to explicitly specify MIGs instead of using auto-discovery ``` # where 'n' is the index, starting at 0 --- set autoscalingGroups[n].name=https://content.googleapis.com/compute/v1/projects/$PROJECTID/zones/$ZONENAME/instanceGroupManagers/$FULL-MIG-NAME,autoscalingGroups[n].maxSize=$MAXSIZE,autoscalingGroups[n].minSize=$MINSIZE +--set autoscalingGroups[n].name=https://content.googleapis.com/compute/v1/projects/$PROJECTID/zones/$ZONENAME/instanceGroupManagers/$FULL-MIG-NAME,autoscalingGroups[n].maxSize=$MAXSIZE,autoscalingGroups[n].minSize=$MINSIZE ``` ### Azure AKS @@ -199,28 +204,31 @@ The following parameters are required: - `magnumClusterName=` and `autoscalingGroups` with the names of node groups and min/max node counts - or `autoDiscovery.clusterName=` with one or more `autoDiscovery.roles`. -Additionally, `cloudConfigPath: "/etc/kubernetes/cloud-config"` must be set as this should be the location -of the cloud-config file on the host. +Additionally, `cloudConfigPath: "/etc/kubernetes/cloud-config"` must be set as this should be the location of the cloud-config file on the host. Example values files can be found [here](../../cluster-autoscaler/cloudprovider/magnum/examples). Install the chart with -``` +```console $ helm install my-release autoscaler/cluster-autoscaler -f myvalues.yaml ``` + ### Cluster-API `cloudProvider: clusterapi` must be set, and then one or more of + - `autoDiscovery.clusterName` - or `autoDiscovery.labels` -See [here](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/clusterapi/README.md#configuring-node-group-auto-discovery) for more details + +See [here](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/clusterapi/README.md#configuring-node-group-auto-discovery) for more details. Additional config parameters available, see the `values.yaml` for more details -`clusterAPIMode` -`clusterAPIKubeconfigSecret` -`clusterAPIWorkloadKubeconfigPath` -`clusterAPICloudConfigPath` + +- `clusterAPIMode` +- `clusterAPIKubeconfigSecret` +- `clusterAPIWorkloadKubeconfigPath` +- `clusterAPICloudConfigPath` ## Uninstalling the Chart @@ -253,7 +261,9 @@ Once you have the IAM role configured, you would then need to `--set rbac.servic ### Azure - Using azure workload identity You can use the project [Azure workload identity](https://github.com/Azure/azure-workload-identity), to automatically configure the correct setup for your pods to used federated identity with Azure. + You can also set the correct settings yourself instead of relying on this project. + For example the following configuration will configure the Autoscaler to use your federated identity: ```yaml @@ -280,8 +290,7 @@ extraVolumeMounts: ## Troubleshooting -The chart will succeed even if the container arguments are incorrect. A few minutes after starting -`kubectl logs -l "app=aws-cluster-autoscaler" --tail=50` should loop through something like +The chart will succeed even if the container arguments are incorrect. A few minutes after starting `kubectl logs -l "app=aws-cluster-autoscaler" --tail=50` should loop through something like ``` polling_autoscaler.go:111] Poll finished @@ -309,6 +318,24 @@ Containers: Though enough for the majority of installations, the default PodSecurityPolicy _could_ be too restrictive depending on the specifics of your release. Please make sure to check that the template fits with any customizations made or disable it by setting `rbac.pspEnabled` to `false`. +### VerticalPodAutoscaler + +The CA Helm Chart can install a [`VerticalPodAutoscaler`](https://github.com/kubernetes/autoscaler/blob/master/vertical-pod-autoscaler/README.md) object from Chart version `9.27.0` +onwards for the Cluster Autoscaler Deployment to scale the CA as appropriate, but for that, we +need to install the VPA to the cluster separately. A VPA can help minimize wasted resources +when usage spikes periodically or remediate containers that are being OOMKilled. + +The following example snippet can be used to install VPA that allows scaling down from the default recommendations of the deployment template: + +```yaml +vpa: + enabled: true + containerPolicy: + minAllowed: + cpu: 20m + memory: 50Mi +``` + ## Values | Key | Type | Default | Description | @@ -355,10 +382,11 @@ Though enough for the majority of installations, the default PodSecurityPolicy _ | extraVolumeSecrets | object | `{}` | Additional volumes to mount from Secrets. | | extraVolumes | list | `[]` | Additional volumes. | | fullnameOverride | string | `""` | String to fully override `cluster-autoscaler.fullname` template. | +| hostNetwork | bool | `false` | Whether to expose network interfaces of the host machine to pods. | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.pullSecrets | list | `[]` | Image pull secrets | -| image.repository | string | `"k8s.gcr.io/autoscaling/cluster-autoscaler"` | Image repository | -| image.tag | string | `"v1.23.0"` | Image tag | +| image.repository | string | `"registry.k8s.io/autoscaling/cluster-autoscaler"` | Image repository | +| image.tag | string | `"v1.27.2"` | Image tag | | kubeTargetVersionOverride | string | `""` | Allow overriding the `.Capabilities.KubeVersion.GitVersion` check. Useful for `helm template` commands. | | magnumCABundlePath | string | `"/etc/kubernetes/ca-bundle.crt"` | Path to the host's CA bundle, from `ca-file` in the cloud-config file. | | magnumClusterName | string | `""` | Cluster name or ID in Magnum. Required if `cloudProvider=magnum` and not setting `autoDiscovery.clusterName`. | @@ -383,6 +411,7 @@ Though enough for the majority of installations, the default PodSecurityPolicy _ | rbac.serviceAccount.name | string | `""` | The name of the ServiceAccount to use. If not set and create is `true`, a name is generated using the fullname template. | | replicaCount | int | `1` | Desired number of pods | | resources | object | `{}` | Pod resource requests and limits. | +| secretKeyRefNameOverride | string | `""` | Overrides the name of the Secret to use when loading the secretKeyRef for AWS and Azure env variables | | securityContext | object | `{}` | [Security context for pod](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) | | service.annotations | object | `{}` | Annotations to add to service | | service.create | bool | `true` | If `true`, a Service will be created. | @@ -403,3 +432,7 @@ Though enough for the majority of installations, the default PodSecurityPolicy _ | tolerations | list | `[]` | List of node taints to tolerate (requires Kubernetes >= 1.6). | | topologySpreadConstraints | list | `[]` | You can use topology spread constraints to control how Pods are spread across your cluster among failure-domains such as regions, zones, nodes, and other user-defined topology domains. (requires Kubernetes >= 1.19). | | updateStrategy | object | `{}` | [Deployment update strategy](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy) | +| vpa | object | `{"containerPolicy":{},"enabled":false,"updateMode":"Auto"}` | Configure a VerticalPodAutoscaler for the cluster-autoscaler Deployment. | +| vpa.containerPolicy | object | `{}` | [ContainerResourcePolicy](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler/v0.13.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L159). The containerName is always et to the deployment's container name. This value is required if VPA is enabled. | +| vpa.enabled | bool | `false` | If true, creates a VerticalPodAutoscaler. | +| vpa.updateMode | string | `"Auto"` | [UpdateMode](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler/v0.13.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L124) | diff --git a/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/README.md.gotmpl b/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/README.md.gotmpl index 80dce493f..611ad6bb1 100644 --- a/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/README.md.gotmpl +++ b/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/README.md.gotmpl @@ -2,20 +2,20 @@ {{ template "chart.description" . }} -## TL;DR: +## TL;DR ```console $ helm repo add autoscaler https://kubernetes.github.io/autoscaler # Method 1 - Using Autodiscovery $ helm install my-release autoscaler/cluster-autoscaler \ ---set 'autoDiscovery.clusterName'= + --set 'autoDiscovery.clusterName'= # Method 2 - Specifying groups manually $ helm install my-release autoscaler/cluster-autoscaler \ ---set "autoscalingGroups[0].name=your-asg-name" \ ---set "autoscalingGroups[0].maxSize=10" \ ---set "autoscalingGroups[0].minSize=1" + --set "autoscalingGroups[0].name=your-asg-name" \ + --set "autoscalingGroups[0].maxSize=10" \ + --set "autoscalingGroups[0].minSize=1" ``` ## Introduction @@ -68,10 +68,10 @@ Either: To create a valid configuration, follow instructions for your cloud provider: -* [AWS](#aws---using-auto-discovery-of-tagged-instance-groups) -* [GCE](#gce) -* [Azure AKS](#azure-aks) -* [OpenStack Magnum](#openstack-magnum) +- [AWS](#aws---using-auto-discovery-of-tagged-instance-groups) +- [GCE](#gce) +- [Azure AKS](#azure-aks) +- [OpenStack Magnum](#openstack-magnum) ### AWS - Using auto-discovery of tagged instance groups @@ -84,13 +84,19 @@ Auto-discovery finds ASGs tags as below and automatically manages them based on - Set (option) `awsAccessKeyID=` and `awsSecretAccessKey=` if you want to [use AWS credentials directly instead of an instance role](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#using-aws-credentials) ```console -$ helm install my-release autoscaler/cluster-autoscaler --set autoDiscovery.clusterName= --set awsRegion= +$ helm install my-release autoscaler/cluster-autoscaler \ + --set autoDiscovery.clusterName= \ + --set awsRegion= ``` Alternatively with your own AWS credentials ```console -$ helm install my-release autoscaler/cluster-autoscaler --set autoDiscovery.clusterName= --set awsRegion= --set awsAccessKeyID= --set awsSecretAccessKey= +$ helm install my-release autoscaler/cluster-autoscaler \ + --set autoDiscovery.clusterName= \ + --set awsRegion= \ + --set awsAccessKeyID= \ + --set awsSecretAccessKey= ``` #### Specifying groups manually @@ -102,15 +108,14 @@ Without autodiscovery, specify an array of elements each containing ASG name, mi ```console $ helm install my-release autoscaler/cluster-autoscaler \ ---set "autoscalingGroups[0].name=your-asg-name" \ ---set "autoscalingGroups[0].maxSize=10" \ ---set "autoscalingGroups[0].minSize=1" + --set "autoscalingGroups[0].name=your-asg-name" \ + --set "autoscalingGroups[0].maxSize=10" \ + --set "autoscalingGroups[0].minSize=1" ``` #### Auto-discovery -For auto-discovery of instances to work, they must be tagged with the keys in `.Values.autoDiscovery.tags`, which by default are -`k8s.io/cluster-autoscaler/enabled` and `k8s.io/cluster-autoscaler/` +For auto-discovery of instances to work, they must be tagged with the keys in `.Values.autoDiscovery.tags`, which by default are `k8s.io/cluster-autoscaler/enabled` and `k8s.io/cluster-autoscaler/`. The value of the tag does not matter, only the key. @@ -147,7 +152,7 @@ spec: In this example you would need to `--set autoDiscovery.clusterName=my.cluster.internal` when installing. -It is not recommended to try to mix this with setting `autoscalingGroups` +It is not recommended to try to mix this with setting `autoscalingGroups`. See [autoscaler AWS documentation](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#auto-discovery-setup) for a more discussion of the setup. @@ -163,9 +168,9 @@ To use Managed Instance Group (MIG) auto-discovery, provide a YAML file setting ```console $ helm install my-release autoscaler/cluster-autoscaler \ ---set "autoscalingGroupsnamePrefix[0].name=your-ig-prefix,autoscalingGroupsnamePrefix[0].maxSize=10,autoscalingGroupsnamePrefi[0].minSize=1" \ ---set autoDiscovery.clusterName= \ ---set cloudProvider=gce + --set "autoscalingGroupsnamePrefix[0].name=your-ig-prefix,autoscalingGroupsnamePrefix[0].maxSize=10,autoscalingGroupsnamePrefi[0].minSize=1" \ + --set autoDiscovery.clusterName= \ + --set cloudProvider=gce ``` Note that `your-ig-prefix` should be a _prefix_ matching one or more MIGs, and _not_ the full name of the MIG. For example, to match multiple instance groups - `k8s-node-group-a-standard`, `k8s-node-group-b-gpu`, you would use a prefix of `k8s-node-group-`. @@ -174,7 +179,7 @@ In the event you want to explicitly specify MIGs instead of using auto-discovery ``` # where 'n' is the index, starting at 0 --- set autoscalingGroups[n].name=https://content.googleapis.com/compute/v1/projects/$PROJECTID/zones/$ZONENAME/instanceGroupManagers/$FULL-MIG-NAME,autoscalingGroups[n].maxSize=$MAXSIZE,autoscalingGroups[n].minSize=$MINSIZE +--set autoscalingGroups[n].name=https://content.googleapis.com/compute/v1/projects/$PROJECTID/zones/$ZONENAME/instanceGroupManagers/$FULL-MIG-NAME,autoscalingGroups[n].maxSize=$MAXSIZE,autoscalingGroups[n].minSize=$MINSIZE ``` ### Azure AKS @@ -199,29 +204,31 @@ The following parameters are required: - `magnumClusterName=` and `autoscalingGroups` with the names of node groups and min/max node counts - or `autoDiscovery.clusterName=` with one or more `autoDiscovery.roles`. -Additionally, `cloudConfigPath: "/etc/kubernetes/cloud-config"` must be set as this should be the location -of the cloud-config file on the host. +Additionally, `cloudConfigPath: "/etc/kubernetes/cloud-config"` must be set as this should be the location of the cloud-config file on the host. Example values files can be found [here](../../cluster-autoscaler/cloudprovider/magnum/examples). Install the chart with -``` +```console $ helm install my-release autoscaler/cluster-autoscaler -f myvalues.yaml ``` + ### Cluster-API `cloudProvider: clusterapi` must be set, and then one or more of + - `autoDiscovery.clusterName` - or `autoDiscovery.labels` -See [here](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/clusterapi/README.md#configuring-node-group-auto-discovery) for more details +See [here](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/clusterapi/README.md#configuring-node-group-auto-discovery) for more details. Additional config parameters available, see the `values.yaml` for more details -`clusterAPIMode` -`clusterAPIKubeconfigSecret` -`clusterAPIWorkloadKubeconfigPath` -`clusterAPICloudConfigPath` + +- `clusterAPIMode` +- `clusterAPIKubeconfigSecret` +- `clusterAPIWorkloadKubeconfigPath` +- `clusterAPICloudConfigPath` ## Uninstalling the Chart @@ -254,7 +261,9 @@ Once you have the IAM role configured, you would then need to `--set rbac.servic ### Azure - Using azure workload identity You can use the project [Azure workload identity](https://github.com/Azure/azure-workload-identity), to automatically configure the correct setup for your pods to used federated identity with Azure. + You can also set the correct settings yourself instead of relying on this project. + For example the following configuration will configure the Autoscaler to use your federated identity: ```yaml @@ -281,8 +290,7 @@ extraVolumeMounts: ## Troubleshooting -The chart will succeed even if the container arguments are incorrect. A few minutes after starting -`kubectl logs -l "app=aws-cluster-autoscaler" --tail=50` should loop through something like +The chart will succeed even if the container arguments are incorrect. A few minutes after starting `kubectl logs -l "app=aws-cluster-autoscaler" --tail=50` should loop through something like ``` polling_autoscaler.go:111] Poll finished @@ -310,4 +318,22 @@ Containers: Though enough for the majority of installations, the default PodSecurityPolicy _could_ be too restrictive depending on the specifics of your release. Please make sure to check that the template fits with any customizations made or disable it by setting `rbac.pspEnabled` to `false`. +### VerticalPodAutoscaler + +The CA Helm Chart can install a [`VerticalPodAutoscaler`](https://github.com/kubernetes/autoscaler/blob/master/vertical-pod-autoscaler/README.md) object from Chart version `9.27.0` +onwards for the Cluster Autoscaler Deployment to scale the CA as appropriate, but for that, we +need to install the VPA to the cluster separately. A VPA can help minimize wasted resources +when usage spikes periodically or remediate containers that are being OOMKilled. + +The following example snippet can be used to install VPA that allows scaling down from the default recommendations of the deployment template: + +```yaml +vpa: + enabled: true + containerPolicy: + minAllowed: + cpu: 20m + memory: 50Mi +``` + {{ template "chart.valuesSection" . }} diff --git a/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/templates/_helpers.tpl b/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/templates/_helpers.tpl index 3fbf00965..944fd1cf6 100644 --- a/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/templates/_helpers.tpl +++ b/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/templates/_helpers.tpl @@ -70,13 +70,10 @@ Return the appropriate apiVersion for podsecuritypolicy. {{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} {{- if semverCompare "<1.10-0" $kubeTargetVersion -}} {{- print "extensions/v1beta1" -}} -{{- if semverCompare ">1.21-0" $kubeTargetVersion -}} -{{- print "policy/v1" -}} {{- else -}} {{- print "policy/v1beta1" -}} {{- end -}} {{- end -}} -{{- end -}} {{/* Return the appropriate apiVersion for podDisruptionBudget. diff --git a/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/templates/clusterrole.yaml b/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/templates/clusterrole.yaml index e3d36557f..4ee33d81b 100644 --- a/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/templates/clusterrole.yaml +++ b/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/templates/clusterrole.yaml @@ -151,7 +151,7 @@ rules: - cluster.x-k8s.io resources: - machinedeployments - - machinedeployments/scale + - machinepools - machines - machinesets verbs: @@ -159,5 +159,14 @@ rules: - list - update - watch + - apiGroups: + - cluster.x-k8s.io + resources: + - machinedeployments/scale + - machinepools/scale + verbs: + - get + - patch + - update {{- end }} {{- end -}} diff --git a/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/templates/deployment.yaml b/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/templates/deployment.yaml index a8d98d9cb..2b5bba272 100644 --- a/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/templates/deployment.yaml +++ b/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/templates/deployment.yaml @@ -42,6 +42,9 @@ spec: {{- if .Values.dnsPolicy }} dnsPolicy: "{{ .Values.dnsPolicy }}" {{- end }} + {{- if .Values.hostNetwork }} + hostNetwork: {{ .Values.hostNetwork }} + {{- end }} containers: - name: {{ template "cluster-autoscaler.name" . }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" @@ -77,7 +80,7 @@ spec: - --node-group-auto-discovery=mig:namePrefix={{ .name }},min={{ .minSize }},max={{ .maxSize }} {{- end }} {{- end }} - {{- if eq .Values.cloudProvider "oci-oke" }} + {{- if eq .Values.cloudProvider "oci" }} {{- if .Values.cloudConfigPath }} - --nodes={{ .minSize }}:{{ .maxSize }}:{{ .name }} - --balance-similar-node-groups @@ -129,36 +132,36 @@ spec: valueFrom: secretKeyRef: key: AwsAccessKeyId - name: {{ template "cluster-autoscaler.fullname" . }} + name: {{ default (include "cluster-autoscaler.fullname" .) .Values.secretKeyRefNameOverride }} {{- end }} {{- if .Values.awsSecretAccessKey }} - name: AWS_SECRET_ACCESS_KEY valueFrom: secretKeyRef: key: AwsSecretAccessKey - name: {{ template "cluster-autoscaler.fullname" . }} + name: {{ default (include "cluster-autoscaler.fullname" .) .Values.secretKeyRefNameOverride }} {{- end }} {{- else if eq .Values.cloudProvider "azure" }} - name: ARM_SUBSCRIPTION_ID valueFrom: secretKeyRef: key: SubscriptionID - name: {{ template "cluster-autoscaler.fullname" . }} + name: {{ default (include "cluster-autoscaler.fullname" .) .Values.secretKeyRefNameOverride }} - name: ARM_RESOURCE_GROUP valueFrom: secretKeyRef: key: ResourceGroup - name: {{ template "cluster-autoscaler.fullname" . }} + name: {{ default (include "cluster-autoscaler.fullname" .) .Values.secretKeyRefNameOverride }} - name: ARM_VM_TYPE valueFrom: secretKeyRef: key: VMType - name: {{ template "cluster-autoscaler.fullname" . }} + name: {{ default (include "cluster-autoscaler.fullname" .) .Values.secretKeyRefNameOverride }} - name: AZURE_CLUSTER_NAME valueFrom: secretKeyRef: key: ClusterName - name: {{ template "cluster-autoscaler.fullname" . }} + name: {{ default (include "cluster-autoscaler.fullname" .) .Values.secretKeyRefNameOverride }} {{- if .Values.azureUseWorkloadIdentityExtension }} - name: ARM_USE_WORKLOAD_IDENTITY_EXTENSION value: "true" @@ -170,22 +173,22 @@ spec: valueFrom: secretKeyRef: key: TenantID - name: {{ template "cluster-autoscaler.fullname" . }} + name: {{ default (include "cluster-autoscaler.fullname" .) .Values.secretKeyRefNameOverride }} - name: ARM_CLIENT_ID valueFrom: secretKeyRef: key: ClientID - name: {{ template "cluster-autoscaler.fullname" . }} + name: {{ default (include "cluster-autoscaler.fullname" .) .Values.secretKeyRefNameOverride }} - name: ARM_CLIENT_SECRET valueFrom: secretKeyRef: key: ClientSecret - name: {{ template "cluster-autoscaler.fullname" . }} + name: {{ default (include "cluster-autoscaler.fullname" .) .Values.secretKeyRefNameOverride }} - name: AZURE_NODE_RESOURCE_GROUP valueFrom: secretKeyRef: key: NodeResourceGroup - name: {{ template "cluster-autoscaler.fullname" . }} + name: {{ default (include "cluster-autoscaler.fullname" .) .Values.secretKeyRefNameOverride }} {{- end }} {{- end }} {{- range $key, $value := .Values.extraEnv }} diff --git a/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/templates/podsecuritypolicy.yaml b/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/templates/podsecuritypolicy.yaml index f72f16201..e3ce59973 100644 --- a/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/templates/podsecuritypolicy.yaml +++ b/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/templates/podsecuritypolicy.yaml @@ -19,7 +19,7 @@ spec: - 'emptyDir' - 'projected' - 'downwardAPI' - hostNetwork: false + hostNetwork: {{ .Values.hostNetwork }} hostIPC: false hostPID: false runAsUser: diff --git a/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/templates/role.yaml b/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/templates/role.yaml index b22fb58be..44b1678af 100644 --- a/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/templates/role.yaml +++ b/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/templates/role.yaml @@ -49,7 +49,7 @@ rules: - cluster.x-k8s.io resources: - machinedeployments - - machinedeployments/scale + - machinepools - machines - machinesets verbs: @@ -57,6 +57,15 @@ rules: - list - update - watch + - apiGroups: + - cluster.x-k8s.io + resources: + - machinedeployments/scale + - machinepools/scale + verbs: + - get + - patch + - update {{- end }} {{- if ( not .Values.rbac.clusterScoped ) }} - apiGroups: diff --git a/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/templates/vpa.yaml b/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/templates/vpa.yaml new file mode 100644 index 000000000..b889beac9 --- /dev/null +++ b/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/templates/vpa.yaml @@ -0,0 +1,20 @@ +{{- if .Values.vpa.enabled -}} +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + labels: +{{ include "cluster-autoscaler.labels" . | indent 4 }} + name: {{ template "cluster-autoscaler.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + targetRef: + apiVersion: {{ template "deployment.apiVersion" . }} + kind: Deployment + name: {{ template "cluster-autoscaler.fullname" . }} + updatePolicy: + updateMode: {{ .Values.vpa.updateMode | quote }} + resourcePolicy: + containerPolicies: + - containerName: {{ template "cluster-autoscaler.name" . }} + {{- .Values.vpa.containerPolicy | toYaml | nindent 6 }} +{{- end -}} diff --git a/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/values.yaml b/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/values.yaml index 44d16c836..b7b39739d 100644 --- a/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/values.yaml +++ b/argocd-helm-charts/cluster-autoscaler/charts/cluster-autoscaler/values.yaml @@ -2,8 +2,11 @@ # affinity -- Affinity for pod assignment affinity: {} +# additionalLabels -- Labels to add to each object of the chart. +additionalLabels: {} + autoDiscovery: - # cloudProviders `aws`, `gce`, `azure`, `magnum` and `clusterapi` `oci-oke` are supported by auto-discovery at this time + # cloudProviders `aws`, `gce`, `azure`, `magnum`, `clusterapi` and `oci` are supported by auto-discovery at this time # AWS: Set tags as described in https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#auto-discovery-setup # autoDiscovery.clusterName -- Enable autodiscovery for `cloudProvider=aws`, for groups matching `autoDiscovery.tags`. @@ -72,6 +75,14 @@ azureClientID: "" # Required if `cloudProvider=azure` azureClientSecret: "" +# azureClusterName -- Azure AKS cluster name. +# Required if `cloudProvider=azure` +azureClusterName: "" + +# azureNodeResourceGroup -- Azure resource group where the cluster's nodes are located, typically set as `MC___`. +# Required if `cloudProvider=azure` +azureNodeResourceGroup: "" + # azureResourceGroup -- Azure resource group that the cluster is located. # Required if `cloudProvider=azure` azureResourceGroup: "" @@ -84,29 +95,33 @@ azureSubscriptionID: "" # Required if `cloudProvider=azure` azureTenantID: "" +# azureUseManagedIdentityExtension -- Whether to use Azure's managed identity extension for credentials. If using MSI, ensure subscription ID, resource group, and azure AKS cluster name are set. You can only use one authentication method at a time, either azureUseWorkloadIdentityExtension or azureUseManagedIdentityExtension should be set. +azureUseManagedIdentityExtension: false + +# azureUseWorkloadIdentityExtension -- Whether to use Azure's workload identity extension for credentials. See the project here: https://github.com/Azure/azure-workload-identity for more details. You can only use one authentication method at a time, either azureUseWorkloadIdentityExtension or azureUseManagedIdentityExtension should be set. +azureUseWorkloadIdentityExtension: false + # azureVMType -- Azure VM type. azureVMType: "AKS" -# azureClusterName -- Azure AKS cluster name. -# Required if `cloudProvider=azure` -azureClusterName: "" - -# azureNodeResourceGroup -- Azure resource group where the cluster's nodes are located, typically set as `MC___`. -# Required if `cloudProvider=azure` -azureNodeResourceGroup: "" +# cloudConfigPath -- Configuration file for cloud provider. +cloudConfigPath: "" -# azureUseWorkloadIdentityExtension -- Whether to use Azure's workload identity extension for credentials. See the project here: https://github.com/Azure/azure-workload-identity for more details. You can only use one authentication method at a time, either azureUseWorkloadIdentityExtension or azureUseManagedIdentityExtension should be set. -azureUseWorkloadIdentityExtension: false +# cloudProvider -- The cloud provider where the autoscaler runs. +# Currently only `gce`, `aws`, `azure`, `magnum` and `clusterapi` are supported. +# `aws` supported for AWS. `gce` for GCE. `azure` for Azure AKS. +# `magnum` for OpenStack Magnum, `clusterapi` for Cluster API. +cloudProvider: aws -# azureUseManagedIdentityExtension -- Whether to use Azure's managed identity extension for credentials. If using MSI, ensure subscription ID, resource group, and azure AKS cluster name are set. You can only use one authentication method at a time, either azureUseWorkloadIdentityExtension or azureUseManagedIdentityExtension should be set. -azureUseManagedIdentityExtension: false +# clusterAPICloudConfigPath -- Path to kubeconfig for connecting to Cluster API Management Cluster, only used if `clusterAPIMode=kubeconfig-kubeconfig or incluster-kubeconfig` +clusterAPICloudConfigPath: /etc/kubernetes/mgmt-kubeconfig -# magnumClusterName -- Cluster name or ID in Magnum. -# Required if `cloudProvider=magnum` and not setting `autoDiscovery.clusterName`. -magnumClusterName: "" +# clusterAPIConfigMapsNamespace -- Namespace on the workload cluster to store Leader election and status configmaps +clusterAPIConfigMapsNamespace: "" -# magnumCABundlePath -- Path to the host's CA bundle, from `ca-file` in the cloud-config file. -magnumCABundlePath: "/etc/kubernetes/ca-bundle.crt" +# clusterAPIKubeconfigSecret -- Secret containing kubeconfig for connecting to Cluster API managed workloadcluster +# Required if `cloudProvider=clusterapi` and `clusterAPIMode=kubeconfig-kubeconfig,kubeconfig-incluster or incluster-kubeconfig` +clusterAPIKubeconfigSecret: "" # clusterAPIMode -- Cluster API mode, see https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/clusterapi/README.md#connecting-cluster-autoscaler-to-cluster-api-management-and-workload-clusters # Syntax: workloadClusterMode-ManagementClusterMode @@ -114,28 +129,9 @@ magnumCABundlePath: "/etc/kubernetes/ca-bundle.crt" # if you dont set `clusterAPIKubeconfigSecret`and thus use an in-cluster config or want to use a non capi generated kubeconfig you must do so for the workload kubeconfig as well clusterAPIMode: incluster-incluster # incluster-incluster, incluster-kubeconfig, kubeconfig-incluster, kubeconfig-kubeconfig, single-kubeconfig -# clusterAPIKubeconfigSecret -- Secret containing kubeconfig for connecting to Cluster API managed workloadcluster -# Required if `cloudProvider=clusterapi` and `clusterAPIMode=kubeconfig-kubeconfig,kubeconfig-incluster or incluster-kubeconfig` -clusterAPIKubeconfigSecret: "" - # clusterAPIWorkloadKubeconfigPath -- Path to kubeconfig for connecting to Cluster API managed workloadcluster, only used if `clusterAPIMode=kubeconfig-kubeconfig or kubeconfig-incluster` clusterAPIWorkloadKubeconfigPath: /etc/kubernetes/value -# clusterAPICloudConfigPath -- Path to kubeconfig for connecting to Cluster API Management Cluster, only used if `clusterAPIMode=kubeconfig-kubeconfig or incluster-kubeconfig` -clusterAPICloudConfigPath: /etc/kubernetes/mgmt-kubeconfig - -# clusterAPIConfigMapsNamespace -- Namespace on the workload cluster to store Leader election and status configmaps -clusterAPIConfigMapsNamespace: "" - -# cloudConfigPath -- Configuration file for cloud provider. -cloudConfigPath: "" - -# cloudProvider -- The cloud provider where the autoscaler runs. -# Currently only `gce`, `aws`, `azure`, `magnum` and `clusterapi` are supported. -# `aws` supported for AWS. `gce` for GCE. `azure` for Azure AKS. -# `magnum` for OpenStack Magnum, `clusterapi` for Cluster API. -cloudProvider: aws - # containerSecurityContext -- [Security context for container](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) containerSecurityContext: {} # capabilities: @@ -151,17 +147,18 @@ deployment: # If autoscaler does not depend on cluster DNS, recommended to set this to `Default`. dnsPolicy: ClusterFirst +# envFromConfigMap -- ConfigMap name to use as envFrom. +envFromConfigMap: "" + +# envFromSecret -- Secret name to use as envFrom. +envFromSecret: "" + ## Priorities Expander # expanderPriorities -- The expanderPriorities is used if `extraArgs.expander` contains `priority` and expanderPriorities is also set with the priorities. # If `extraArgs.expander` contains `priority`, then expanderPriorities is used to define cluster-autoscaler-priority-expander priorities. # See: https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/expander/priority/readme.md expanderPriorities: {} -# priorityConfigMapAnnotations -- Annotations to add to `cluster-autoscaler-priority-expander` ConfigMap. -priorityConfigMapAnnotations: {} - # key1: "value1" - # key2: "value2" - # extraArgs -- Additional container arguments. # Refer to https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/FAQ.md#what-are-the-parameters-to-ca for the full list of cluster autoscaler # parameters and their default values. @@ -200,11 +197,17 @@ extraEnvConfigMaps: {} # extraEnvSecrets -- Additional container environment variables from Secrets. extraEnvSecrets: {} -# envFromConfigMap -- ConfigMap name to use as envFrom. -envFromConfigMap: "" +# extraVolumeMounts -- Additional volumes to mount. +extraVolumeMounts: [] + # - name: ssl-certs + # mountPath: /etc/ssl/certs/ca-certificates.crt + # readOnly: true -# envFromSecret -- Secret name to use as envFrom. -envFromSecret: "" +# extraVolumes -- Additional volumes. +extraVolumes: [] + # - name: ssl-certs + # hostPath: + # path: /etc/ssl/certs/ca-bundle.crt # extraVolumeSecrets -- Additional volumes to mount from Secrets. extraVolumeSecrets: {} @@ -217,26 +220,17 @@ extraVolumeSecrets: {} # - key: subkey # path: mypath -# extraVolumes -- Additional volumes. -extraVolumes: [] - # - name: ssl-certs - # hostPath: - # path: /etc/ssl/certs/ca-bundle.crt - -# extraVolumeMounts -- Additional volumes to mount. -extraVolumeMounts: [] - # - name: ssl-certs - # mountPath: /etc/ssl/certs/ca-certificates.crt - # readOnly: true - # fullnameOverride -- String to fully override `cluster-autoscaler.fullname` template. fullnameOverride: "" +# hostNetwork -- Whether to expose network interfaces of the host machine to pods. +hostNetwork: false + image: # image.repository -- Image repository - repository: k8s.gcr.io/autoscaling/cluster-autoscaler + repository: registry.k8s.io/autoscaling/cluster-autoscaler # image.tag -- Image tag - tag: v1.23.0 + tag: v1.27.2 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent ## Optionally specify an array of imagePullSecrets. @@ -250,6 +244,13 @@ image: # kubeTargetVersionOverride -- Allow overriding the `.Capabilities.KubeVersion.GitVersion` check. Useful for `helm template` commands. kubeTargetVersionOverride: "" +# magnumCABundlePath -- Path to the host's CA bundle, from `ca-file` in the cloud-config file. +magnumCABundlePath: "/etc/kubernetes/ca-bundle.crt" + +# magnumClusterName -- Cluster name or ID in Magnum. +# Required if `cloudProvider=magnum` and not setting `autoDiscovery.clusterName`. +magnumClusterName: "" + # nameOverride -- String to partially override `cluster-autoscaler.fullname` template (will maintain the release name) nameOverride: "" @@ -267,12 +268,29 @@ podDisruptionBudget: # podLabels -- Labels to add to each pod. podLabels: {} -# additionalLabels -- Labels to add to each object of the chart. -additionalLabels: {} - # priorityClassName -- priorityClassName priorityClassName: "system-cluster-critical" +# priorityConfigMapAnnotations -- Annotations to add to `cluster-autoscaler-priority-expander` ConfigMap. +priorityConfigMapAnnotations: {} + # key1: "value1" + # key2: "value2" + +## Custom PrometheusRule to be defined +## The value is evaluated as a template, so, for example, the value can depend on .Release or .Chart +## ref: https://github.com/coreos/prometheus-operator#customresourcedefinitions +prometheusRule: + # prometheusRule.enabled -- If true, creates a Prometheus Operator PrometheusRule. + enabled: false + # prometheusRule.additionalLabels -- Additional labels to be set in metadata. + additionalLabels: {} + # prometheusRule.namespace -- Namespace which Prometheus is running in. + namespace: monitoring + # prometheusRule.interval -- How often rules in the group are evaluated (falls back to `global.evaluation_interval` if not set). + interval: null + # prometheusRule.rules -- Rules spec template (see https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#rule). + rules: [] + rbac: # rbac.create -- If `true`, create and use RBAC resources. create: true @@ -351,21 +369,6 @@ serviceMonitor: # serviceMonitor.metricRelabelings -- MetricRelabelConfigs to apply to samples before ingestion. metricRelabelings: {} -## Custom PrometheusRule to be defined -## The value is evaluated as a template, so, for example, the value can depend on .Release or .Chart -## ref: https://github.com/coreos/prometheus-operator#customresourcedefinitions -prometheusRule: - # prometheusRule.enabled -- If true, creates a Prometheus Operator PrometheusRule. - enabled: false - # prometheusRule.additionalLabels -- Additional labels to be set in metadata. - additionalLabels: {} - # prometheusRule.namespace -- Namespace which Prometheus is running in. - namespace: monitoring - # prometheusRule.interval -- How often rules in the group are evaluated (falls back to `global.evaluation_interval` if not set). - interval: null - # prometheusRule.rules -- Rules spec template (see https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#rule). - rules: [] - # tolerations -- List of node taints to tolerate (requires Kubernetes >= 1.6). tolerations: [] @@ -384,3 +387,15 @@ updateStrategy: {} # maxSurge: 1 # maxUnavailable: 0 # type: RollingUpdate + +# vpa -- Configure a VerticalPodAutoscaler for the cluster-autoscaler Deployment. +vpa: + # vpa.enabled -- If true, creates a VerticalPodAutoscaler. + enabled: false + # vpa.updateMode -- [UpdateMode](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler/v0.13.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L124) + updateMode: "Auto" + # vpa.containerPolicy -- [ContainerResourcePolicy](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler/v0.13.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L159). The containerName is always et to the deployment's container name. This value is required if VPA is enabled. + containerPolicy: {} + +# secretKeyRefNameOverride -- Overrides the name of the Secret to use when loading the secretKeyRef for AWS and Azure env variables +secretKeyRefNameOverride: ""