From 726a1c648d794467c76347ca90b6ba4944e6032d Mon Sep 17 00:00:00 2001 From: Archisman Date: Mon, 11 Nov 2024 21:03:35 +0530 Subject: [PATCH] Support Cluster API - Cluster AutoScaler integration Signed-off-by: Archisman --- .../charts/aws/templates/AWSMachinePool.yaml | 18 ++++---- .../aws/templates/AWSMachineTemplate.yaml | 15 ++++--- .../charts/aws/templates/KubeadmConfig.yaml | 8 ++-- .../aws/templates/KubeadmConfigTemplate.yaml | 10 ++++- .../aws/templates/MachineDeployment.yaml | 34 +++++++++++---- .../charts/aws/templates/MachinePool.yaml | 19 ++++---- .../capi-cluster/charts/aws/values.yaml | 24 +---------- .../examples/aws/cluster-autoscaler.yaml | 40 +++++++++++++++++ .../examples/aws/machinepools.yaml | 43 +++++++++++++++++++ .../capi-cluster/templates/provider-aws.yaml | 4 +- argocd-helm-charts/capi-cluster/values.yaml | 3 +- 11 files changed, 154 insertions(+), 64 deletions(-) create mode 100644 argocd-helm-charts/capi-cluster/examples/aws/cluster-autoscaler.yaml create mode 100644 argocd-helm-charts/capi-cluster/examples/aws/machinepools.yaml diff --git a/argocd-helm-charts/capi-cluster/charts/aws/templates/AWSMachinePool.yaml b/argocd-helm-charts/capi-cluster/charts/aws/templates/AWSMachinePool.yaml index 32b10b325..f2268f74d 100644 --- a/argocd-helm-charts/capi-cluster/charts/aws/templates/AWSMachinePool.yaml +++ b/argocd-helm-charts/capi-cluster/charts/aws/templates/AWSMachinePool.yaml @@ -1,20 +1,20 @@ -{{- if (.Values.global.capa).autoScaler }} -{{- range $index, $machinePool := $.Values.machinePools }} +{{- if (.Values.global.capa).enableMachinePools }} +{{- range $index, $nodeGroup := $.Values.nodeGroups }} --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSMachinePool metadata: - name: {{ printf "%s-%s" $.Values.global.clusterName $machinePool.name }} + name: {{ printf "%s-%s" $.Values.global.clusterName $nodeGroup.name }} spec: - minSize: {{ $machinePool.minSize | default 1 }} - maxSize: {{ $machinePool.maxSize | default 3 }} + minSize: {{ $nodeGroup.minSize | default 1 }} + maxSize: {{ $nodeGroup.maxSize | default 3 }} awsLaunchTemplate: - ami: {{ toYaml $machinePool.ami | nindent 6 }} + ami: {{ toYaml $nodeGroup.ami | nindent 6 }} iamInstanceProfile: nodes.cluster-api-provider-aws.sigs.k8s.io - instanceType: {{ $machinePool.instanceType }} - sshKeyName: {{ $machinePool.sshKeyName }} + instanceType: {{ $nodeGroup.instanceType }} + sshKeyName: {{ $nodeGroup.sshKeyName }} imageLookupBaseOS: ubuntu-20.04 rootVolume: - size: {{ $machinePool.rootVolumeSize | default 35 }} + size: {{ $nodeGroup.rootVolumeSize | default 35 }} {{- end }} {{- end }} diff --git a/argocd-helm-charts/capi-cluster/charts/aws/templates/AWSMachineTemplate.yaml b/argocd-helm-charts/capi-cluster/charts/aws/templates/AWSMachineTemplate.yaml index fd7075dcf..b2e5205b1 100644 --- a/argocd-helm-charts/capi-cluster/charts/aws/templates/AWSMachineTemplate.yaml +++ b/argocd-helm-charts/capi-cluster/charts/aws/templates/AWSMachineTemplate.yaml @@ -14,21 +14,22 @@ spec: rootVolume: size: {{ .Values.controlPlane.rootVolumeSize | default 35 }} -{{- if (not (.Values.global.capa).autoScaler) }} +{{- if (.Values.global).enableClusterAutoscaler }} +{{- range $index, $nodeGroup := $.Values.nodeGroups }} --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSMachineTemplate metadata: - name: {{ .Values.global.clusterName }}-md-0 + name: {{ printf "%s-%s" $.Values.global.clusterName $nodeGroup.name }} spec: template: spec: + ami: {{ toYaml $nodeGroup.ami | nindent 8 }} iamInstanceProfile: nodes.cluster-api-provider-aws.sigs.k8s.io - instanceType: {{ .Values.workerNodes.instanceType }} - sshKeyName: {{ .Values.workerNodes.sshKeyName }} + instanceType: {{ $nodeGroup.instanceType }} + sshKeyName: {{ $nodeGroup.sshKeyName }} imageLookupBaseOS: ubuntu-20.04 - publicIP: {{ .Values.workerNodes.publicIP }} - ami: {{ toYaml .Values.workerNodes.ami | nindent 8 }} rootVolume: - size: {{ .Values.workerNodes.rootVolumeSize | default 35 }} + size: {{ $nodeGroup.rootVolumeSize | default 35 }} +{{- end }} {{- end }} diff --git a/argocd-helm-charts/capi-cluster/charts/aws/templates/KubeadmConfig.yaml b/argocd-helm-charts/capi-cluster/charts/aws/templates/KubeadmConfig.yaml index 7d46812ed..3d8f11845 100644 --- a/argocd-helm-charts/capi-cluster/charts/aws/templates/KubeadmConfig.yaml +++ b/argocd-helm-charts/capi-cluster/charts/aws/templates/KubeadmConfig.yaml @@ -1,18 +1,18 @@ {{- if (.Values.global.capa).autoScaler }} -{{- range $index, $machinePool := $.Values.machinePools }} +{{- range $index, $nodeGroup := $.Values.nodeGroups }} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfig metadata: - name: {{ printf "%s-%s" $.Values.global.clusterName $machinePool.name }} + name: {{ printf "%s-%s" $.Values.global.clusterName $nodeGroup.name }} spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: cloud-provider: external name: '{{"{{"}} ds.meta_data.local_hostname {{"}}"}}' - {{- if $machinePool.taints }} - taints: {{- toYaml $machinePool.taints | nindent 8 }} + {{- if $nodeGroup.taints }} + taints: {{- toYaml $nodeGroup.taints | nindent 8 }} {{- end }} {{- end }} {{- end }} diff --git a/argocd-helm-charts/capi-cluster/charts/aws/templates/KubeadmConfigTemplate.yaml b/argocd-helm-charts/capi-cluster/charts/aws/templates/KubeadmConfigTemplate.yaml index 38115a7c3..f1d95f2eb 100644 --- a/argocd-helm-charts/capi-cluster/charts/aws/templates/KubeadmConfigTemplate.yaml +++ b/argocd-helm-charts/capi-cluster/charts/aws/templates/KubeadmConfigTemplate.yaml @@ -1,8 +1,10 @@ -{{- if (not (.Values.global.capa).autoScaler) }} +{{- if (.Values.global.capa).enableClusterAutoscaler }} +{{- range $index, $nodeGroup := $.Values.nodeGroups }} +--- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate metadata: - name: {{ .Values.global.clusterName }} + name: {{ printf "%s-%s" $.Values.global.clusterName $nodeGroup.name }} spec: template: spec: @@ -11,4 +13,8 @@ spec: kubeletExtraArgs: cloud-provider: external name: '{{"{{"}} ds.meta_data.local_hostname {{"}}"}}' + {{- if $nodeGroup.taints }} + taints: {{- toYaml $nodeGroup.taints | nindent 8 }} + {{- end }} +{{- end }} {{- end }} diff --git a/argocd-helm-charts/capi-cluster/charts/aws/templates/MachineDeployment.yaml b/argocd-helm-charts/capi-cluster/charts/aws/templates/MachineDeployment.yaml index ba06deee3..455764b1e 100644 --- a/argocd-helm-charts/capi-cluster/charts/aws/templates/MachineDeployment.yaml +++ b/argocd-helm-charts/capi-cluster/charts/aws/templates/MachineDeployment.yaml @@ -1,22 +1,40 @@ -{{- if (not (.Values.global.capa).autoScaler) }} +{{- if (.Values.global).enableClusterAutoscaler }} +{{- range $index, $nodeGroup := $.Values.nodeGroups }} +--- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: - name: {{ .Values.global.clusterName }}-md-0 + name: {{ printf "%s-%s" $.Values.global.clusterName $nodeGroup.name }} + annotations: + cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: {{ $nodeGroup.minSize | default 1 | quote }} + cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: {{ $nodeGroup.maxSize | default 3 | quote }} spec: - clusterName: {{ .Values.global.clusterName }} - replicas: {{ .Values.workerNodes.replicas }} + clusterName: {{ $.Values.global.clusterName }} + replicas: {{ $nodeGroup.replicas | default 2 }} template: + {{- if $nodeGroup.labels }} + metadata: + # Label should meet one of the following criterias to propagate to Node : + # + # (1) Has node-role.kubernetes.io as prefix. + # (2) Belongs to node-restriction.kubernetes.io domain. + # (3) Belongs to node.cluster.x-k8s.io domain. + # + # REFER : https://cluster-api.sigs.k8s.io/developer/architecture/controllers/metadata-propagation#machine + labels: {{- toYaml $nodeGroup.labels | nindent 8 }} + {{- end }} + spec: + clusterName: {{ $.Values.global.clusterName }} bootstrap: configRef: apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate - name: {{ .Values.global.clusterName }} - clusterName: {{ .Values.global.clusterName }} + name: {{ printf "%s-%s" $.Values.global.clusterName $nodeGroup.name }} infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSMachineTemplate - name: {{ .Values.global.clusterName }}-md-0 - version: {{ .Values.global.kubernetes.version }} + name: {{ printf "%s-%s" $.Values.global.clusterName $nodeGroup.name }} + version: {{ $.Values.global.kubernetes.version }} +{{- end }} {{- end }} diff --git a/argocd-helm-charts/capi-cluster/charts/aws/templates/MachinePool.yaml b/argocd-helm-charts/capi-cluster/charts/aws/templates/MachinePool.yaml index dc4653288..75d7434a9 100644 --- a/argocd-helm-charts/capi-cluster/charts/aws/templates/MachinePool.yaml +++ b/argocd-helm-charts/capi-cluster/charts/aws/templates/MachinePool.yaml @@ -1,17 +1,17 @@ -{{- if (.Values.global.capa).autoScaler }} -{{- range $index, $machinePool := $.Values.machinePools }} +{{- if (.Values.global.capa).enableMachinePools }} +{{- range $index, $nodeGroup := $.Values.nodeGroups }} --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachinePool metadata: - name: {{ printf "%s-%s" $.Values.global.clusterName $machinePool.name }} + name: {{ printf "%s-%s" $.Values.global.clusterName $nodeGroup.name }} annotations: cluster.x-k8s.io/replicas-managed-by: external-autoscaler spec: clusterName: {{ $.Values.global.clusterName }} - replicas: {{ $machinePool.replicas }} + replicas: {{ $nodeGroup.replicas | default 2 }} template: - {{- if $machinePool.labels }} + {{- if $nodeGroup.labels }} metadata: # Label should meet one of the following criterias to propagate to Node : # @@ -20,19 +20,22 @@ spec: # (3) Belongs to node.cluster.x-k8s.io domain. # # REFER : https://cluster-api.sigs.k8s.io/developer/architecture/controllers/metadata-propagation#machine - labels: {{- toYaml $machinePool.labels | nindent 8 }} + # + # NOTE : The 'MachinePool Machine' proposal implementation needs to get merged for this to + # work : https://github.com/kubernetes-sigs/cluster-api-provider-aws/pull/5174 + labels: {{- toYaml $nodeGroup.labels | nindent 8 }} {{- end }} spec: bootstrap: configRef: apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfig - name: {{ printf "%s-%s" $.Values.global.clusterName $machinePool.name }} + name: {{ printf "%s-%s" $.Values.global.clusterName $nodeGroup.name }} clusterName: {{ $.Values.global.clusterName }} infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSMachinePool - name: {{ printf "%s-%s" $.Values.global.clusterName $machinePool.name }} + name: {{ printf "%s-%s" $.Values.global.clusterName $nodeGroup.name }} version: {{ $.Values.global.kubernetes.version }} {{- end }} {{- end }} diff --git a/argocd-helm-charts/capi-cluster/charts/aws/values.yaml b/argocd-helm-charts/capi-cluster/charts/aws/values.yaml index 782b8d0f4..c4343d773 100644 --- a/argocd-helm-charts/capi-cluster/charts/aws/values.yaml +++ b/argocd-helm-charts/capi-cluster/charts/aws/values.yaml @@ -1,6 +1,4 @@ -secretName: capi-cluster-token -sshKeyName: kubeaid-demo -region: us-east-2 +secretName: cloud-credentials bastion: enabled: true vpc: @@ -10,23 +8,3 @@ pods: controlPlane: loadBalancerType: nlb - region: us-east-2 - replicas: 1 - instanceType: t2.medium - ami: - id: ami-05145146e3a9db6f3 - -machinePools: - - name: primary - replicas: 1 - region: us-east-2 - instanceType: t2.medium - sshKeyName: kubeaid-demo - ami: - id: ami-05145146e3a9db6f3 - - name: secondary - replicas: 1 - instanceType: t2.medium - sshKeyName: kubeaid-demo - ami: - id: ami-05145146e3a9db6f3 diff --git a/argocd-helm-charts/capi-cluster/examples/aws/cluster-autoscaler.yaml b/argocd-helm-charts/capi-cluster/examples/aws/cluster-autoscaler.yaml new file mode 100644 index 000000000..e0e57a4ad --- /dev/null +++ b/argocd-helm-charts/capi-cluster/examples/aws/cluster-autoscaler.yaml @@ -0,0 +1,40 @@ +global: + clusterName: kubeaid-demo + kubeaidConfig: + repo: https://github.com/Archisman-Mridha/kubeaid-config + +provider: + aws: true + +aws: + secretName: cloud-credentials + sshKeyName: kubeaid-demo + region: us-east-1 + bastion: + enabled: true + vpc: + cidrBlock: 10.14.0.0/22 + pods: + cidrBlock: 10.244.0.0/16 + controlPlane: + loadBalancer: + loadBalancerType: nlb + region: us-east-1 + replicas: 1 + instanceType: t4g.medium + ami: + id: ami-0fc044fa0d061d18d + nodeGroups: + - name: primary + # minSize: 1 + # maxSize: 3 + # replicas: 2 + instanceType: t4g.medium + sshKeyName: kubeaid-demo + ami: + id: ami-0fc044fa0d061d18d + rootVolumeSize: 35 + labels: + node.cluster.x-k8s.io/nodegroup: primary + node-role.kubernetes.io/bootstrapper: "" + taints: [] diff --git a/argocd-helm-charts/capi-cluster/examples/aws/machinepools.yaml b/argocd-helm-charts/capi-cluster/examples/aws/machinepools.yaml new file mode 100644 index 000000000..2d9200e12 --- /dev/null +++ b/argocd-helm-charts/capi-cluster/examples/aws/machinepools.yaml @@ -0,0 +1,43 @@ +global: + clusterName: kubeaid-demo + enableClusterAutoscaler: false + capa: + enableMachinePools: true + kubeaidConfig: + repo: https://github.com/Archisman-Mridha/kubeaid-config + +provider: + aws: true + +aws: + secretName: cloud-credentials + sshKeyName: kubeaid-demo + region: us-east-1 + bastion: + enabled: true + vpc: + cidrBlock: 10.14.0.0/22 + pods: + cidrBlock: 10.244.0.0/16 + controlPlane: + loadBalancer: + loadBalancerType: nlb + region: us-east-1 + replicas: 1 + instanceType: t4g.medium + ami: + id: ami-0fc044fa0d061d18d + nodeGroups: + - name: primary + # minSize: 1 + # maxSize: 3 + # replicas: 2 + instanceType: t4g.medium + sshKeyName: kubeaid-demo + ami: + id: ami-0fc044fa0d061d18d + rootVolumeSize: 35 + labels: + node.cluster.x-k8s.io/nodegroup: primary + node-role.kubernetes.io/bootstrapper: "" + taints: [] diff --git a/argocd-helm-charts/capi-cluster/templates/provider-aws.yaml b/argocd-helm-charts/capi-cluster/templates/provider-aws.yaml index 8e202b5b2..ecb870721 100644 --- a/argocd-helm-charts/capi-cluster/templates/provider-aws.yaml +++ b/argocd-helm-charts/capi-cluster/templates/provider-aws.yaml @@ -1,10 +1,10 @@ {{- if (.Values.provider).aws }} -# For non Obmondo users, no customerid is needed. +{{- /* For non Obmondo users, no customerid is needed. */ -}} {{- $name := "aws" }} {{- $namespace := "capi-cluster" }} -# For obmondo customers. +{{- /* For obmondo customers. */ -}} {{- if .Values.global.customerid }} {{- $name = printf "%s-%s" $name .Values.global.customerid }} {{- $namespace = printf "%s-%s" $namespace .Values.global.customerid }} diff --git a/argocd-helm-charts/capi-cluster/values.yaml b/argocd-helm-charts/capi-cluster/values.yaml index 335498333..aa65c7c03 100644 --- a/argocd-helm-charts/capi-cluster/values.yaml +++ b/argocd-helm-charts/capi-cluster/values.yaml @@ -3,9 +3,10 @@ global: version: v1.31.0 clusterAPI: version: v1.8.4 + enableClusterAutoscaler: true capa: version: v2.7.1 - autoScaler: true + enableMachinePools: false caph: version: v1.0.0-beta.43 capz: