Support Cluster API - Cluster AutoScaler integration

Signed-off-by: Archisman <[email protected]>
blackwoodseven · Nov 11, 2024 · 726a1c6 · 726a1c6
1 parent 4af97fd
commit 726a1c6
Show file tree

Hide file tree

Showing 11 changed files with 154 additions and 64 deletions.
diff --git a/argocd-helm-charts/capi-cluster/charts/aws/templates/AWSMachinePool.yaml b/argocd-helm-charts/capi-cluster/charts/aws/templates/AWSMachinePool.yaml
@@ -1,20 +1,20 @@
-{{- if (.Values.global.capa).autoScaler }}
-{{- range $index, $machinePool := $.Values.machinePools }}
+{{- if (.Values.global.capa).enableMachinePools }}
+{{- range $index, $nodeGroup := $.Values.nodeGroups }}
 ---
 apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
 kind: AWSMachinePool
 metadata:
-  name: {{ printf "%s-%s" $.Values.global.clusterName $machinePool.name }}
+  name: {{ printf "%s-%s" $.Values.global.clusterName $nodeGroup.name }}
 spec:
-  minSize: {{ $machinePool.minSize | default 1 }}
-  maxSize: {{ $machinePool.maxSize | default 3 }}
+  minSize: {{ $nodeGroup.minSize | default 1 }}
+  maxSize: {{ $nodeGroup.maxSize | default 3 }}
   awsLaunchTemplate:
-    ami: {{ toYaml $machinePool.ami | nindent 6 }}
+    ami: {{ toYaml $nodeGroup.ami | nindent 6 }}
     iamInstanceProfile: nodes.cluster-api-provider-aws.sigs.k8s.io
-    instanceType: {{ $machinePool.instanceType }}
-    sshKeyName: {{ $machinePool.sshKeyName }}
+    instanceType: {{ $nodeGroup.instanceType }}
+    sshKeyName: {{ $nodeGroup.sshKeyName }}
     imageLookupBaseOS: ubuntu-20.04
     rootVolume:
-      size: {{ $machinePool.rootVolumeSize | default 35 }}
+      size: {{ $nodeGroup.rootVolumeSize | default 35 }}
 {{- end }}
 {{- end }}
diff --git a/argocd-helm-charts/capi-cluster/charts/aws/templates/AWSMachineTemplate.yaml b/argocd-helm-charts/capi-cluster/charts/aws/templates/AWSMachineTemplate.yaml
@@ -14,21 +14,22 @@ spec:
       rootVolume:
         size: {{ .Values.controlPlane.rootVolumeSize | default 35 }}
 
-{{- if (not (.Values.global.capa).autoScaler) }}
+{{- if (.Values.global).enableClusterAutoscaler }}
+{{- range $index, $nodeGroup := $.Values.nodeGroups }}
 ---
 apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
 kind: AWSMachineTemplate
 metadata:
-  name: {{ .Values.global.clusterName }}-md-0
+  name: {{ printf "%s-%s" $.Values.global.clusterName $nodeGroup.name }}
 spec:
   template:
     spec:
+      ami: {{ toYaml $nodeGroup.ami | nindent 8 }}
       iamInstanceProfile: nodes.cluster-api-provider-aws.sigs.k8s.io
-      instanceType: {{ .Values.workerNodes.instanceType }}
-      sshKeyName: {{ .Values.workerNodes.sshKeyName }}
+      instanceType: {{ $nodeGroup.instanceType }}
+      sshKeyName: {{ $nodeGroup.sshKeyName }}
       imageLookupBaseOS: ubuntu-20.04
-      publicIP: {{ .Values.workerNodes.publicIP }}
-      ami: {{ toYaml .Values.workerNodes.ami | nindent 8 }}
       rootVolume:
-        size: {{ .Values.workerNodes.rootVolumeSize | default 35 }}
+        size: {{ $nodeGroup.rootVolumeSize | default 35 }}
+{{- end }}
 {{- end }}
diff --git a/argocd-helm-charts/capi-cluster/charts/aws/templates/KubeadmConfig.yaml b/argocd-helm-charts/capi-cluster/charts/aws/templates/KubeadmConfig.yaml
@@ -1,18 +1,18 @@
 {{- if (.Values.global.capa).autoScaler }}
-{{- range $index, $machinePool := $.Values.machinePools }}
+{{- range $index, $nodeGroup := $.Values.nodeGroups }}
 ---
 apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
 kind: KubeadmConfig
 metadata:
-  name: {{ printf "%s-%s" $.Values.global.clusterName $machinePool.name }}
+  name: {{ printf "%s-%s" $.Values.global.clusterName $nodeGroup.name }}
 spec:
   joinConfiguration:
     nodeRegistration:
       kubeletExtraArgs:
         cloud-provider: external
       name: '{{"{{"}} ds.meta_data.local_hostname {{"}}"}}'
-      {{- if $machinePool.taints }}
-      taints: {{- toYaml $machinePool.taints | nindent 8 }}
+      {{- if $nodeGroup.taints }}
+      taints: {{- toYaml $nodeGroup.taints | nindent 8 }}
       {{- end }}
 {{- end }}
 {{- end }}
diff --git a/argocd-helm-charts/capi-cluster/charts/aws/templates/KubeadmConfigTemplate.yaml b/argocd-helm-charts/capi-cluster/charts/aws/templates/KubeadmConfigTemplate.yaml
@@ -1,8 +1,10 @@
-{{- if (not (.Values.global.capa).autoScaler) }}
+{{- if (.Values.global.capa).enableClusterAutoscaler }}
+{{- range $index, $nodeGroup := $.Values.nodeGroups }}
+---
 apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
 kind: KubeadmConfigTemplate
 metadata:
-  name: {{ .Values.global.clusterName }}
+  name: {{ printf "%s-%s" $.Values.global.clusterName $nodeGroup.name }}
 spec:
   template:
     spec:
@@ -11,4 +13,8 @@ spec:
           kubeletExtraArgs:
             cloud-provider: external
           name: '{{"{{"}} ds.meta_data.local_hostname {{"}}"}}'
+          {{- if $nodeGroup.taints }}
+          taints: {{- toYaml $nodeGroup.taints | nindent 8 }}
+          {{- end }}
+{{- end }}
 {{- end }}
diff --git a/argocd-helm-charts/capi-cluster/charts/aws/templates/MachineDeployment.yaml b/argocd-helm-charts/capi-cluster/charts/aws/templates/MachineDeployment.yaml
@@ -1,22 +1,40 @@
-{{- if (not (.Values.global.capa).autoScaler) }}
+{{- if (.Values.global).enableClusterAutoscaler }}
+{{- range $index, $nodeGroup := $.Values.nodeGroups }}
+---
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: MachineDeployment
 metadata:
-  name: {{ .Values.global.clusterName }}-md-0
+  name: {{ printf "%s-%s" $.Values.global.clusterName $nodeGroup.name }}
+  annotations:
+    cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: {{ $nodeGroup.minSize | default 1 | quote }}
+    cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: {{ $nodeGroup.maxSize | default 3 | quote }}
 spec:
-  clusterName: {{ .Values.global.clusterName }}
-  replicas: {{ .Values.workerNodes.replicas }}
+  clusterName: {{ $.Values.global.clusterName }}
+  replicas: {{ $nodeGroup.replicas | default 2 }}
   template:
+    {{- if $nodeGroup.labels }}
+    metadata:
+      # Label should meet one of the following criterias to propagate to Node :
+      #
+      # (1) Has node-role.kubernetes.io as prefix.
+      # (2) Belongs to node-restriction.kubernetes.io domain.
+      # (3) Belongs to node.cluster.x-k8s.io domain.
+      #
+      # REFER : https://cluster-api.sigs.k8s.io/developer/architecture/controllers/metadata-propagation#machine
+      labels: {{- toYaml $nodeGroup.labels | nindent 8 }}
+    {{- end }}
+
     spec:
+      clusterName: {{ $.Values.global.clusterName }}
       bootstrap:
         configRef:
           apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
           kind: KubeadmConfigTemplate
-          name: {{ .Values.global.clusterName }}
-      clusterName: {{ .Values.global.clusterName }}
+          name: {{ printf "%s-%s" $.Values.global.clusterName $nodeGroup.name }}
       infrastructureRef:
         apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
         kind: AWSMachineTemplate
-        name: {{ .Values.global.clusterName }}-md-0
-      version: {{ .Values.global.kubernetes.version }}
+        name: {{ printf "%s-%s" $.Values.global.clusterName $nodeGroup.name }}
+      version: {{ $.Values.global.kubernetes.version }}
+{{- end }}
 {{- end }}
diff --git a/argocd-helm-charts/capi-cluster/charts/aws/templates/MachinePool.yaml b/argocd-helm-charts/capi-cluster/charts/aws/templates/MachinePool.yaml
@@ -1,17 +1,17 @@
-{{- if (.Values.global.capa).autoScaler }}
-{{- range $index, $machinePool := $.Values.machinePools }}
+{{- if (.Values.global.capa).enableMachinePools }}
+{{- range $index, $nodeGroup := $.Values.nodeGroups }}
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: MachinePool
 metadata:
-  name: {{ printf "%s-%s" $.Values.global.clusterName $machinePool.name }}
+  name: {{ printf "%s-%s" $.Values.global.clusterName $nodeGroup.name }}
   annotations:
     cluster.x-k8s.io/replicas-managed-by: external-autoscaler
 spec:
   clusterName: {{ $.Values.global.clusterName }}
-  replicas: {{ $machinePool.replicas }}
+  replicas: {{ $nodeGroup.replicas | default 2 }}
   template:
-    {{- if $machinePool.labels }}
+    {{- if $nodeGroup.labels }}
     metadata:
       # Label should meet one of the following criterias to propagate to Node :
       #
@@ -20,19 +20,22 @@ spec:
       # (3) Belongs to node.cluster.x-k8s.io domain.
       #
       # REFER : https://cluster-api.sigs.k8s.io/developer/architecture/controllers/metadata-propagation#machine
-      labels: {{- toYaml $machinePool.labels | nindent 8 }}
+      #
+      # NOTE : The 'MachinePool Machine' proposal implementation needs to get merged for this to
+      # work : https://github.com/kubernetes-sigs/cluster-api-provider-aws/pull/5174
+      labels: {{- toYaml $nodeGroup.labels | nindent 8 }}
     {{- end }}
     spec:
       bootstrap:
         configRef:
           apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
           kind: KubeadmConfig
-          name: {{ printf "%s-%s" $.Values.global.clusterName $machinePool.name }}
+          name: {{ printf "%s-%s" $.Values.global.clusterName $nodeGroup.name }}
       clusterName: {{ $.Values.global.clusterName }}
       infrastructureRef:
         apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
         kind: AWSMachinePool
-        name: {{ printf "%s-%s" $.Values.global.clusterName $machinePool.name }}
+        name: {{ printf "%s-%s" $.Values.global.clusterName $nodeGroup.name }}
       version: {{ $.Values.global.kubernetes.version }}
 {{- end }}
 {{- end }}
diff --git a/argocd-helm-charts/capi-cluster/charts/aws/values.yaml b/argocd-helm-charts/capi-cluster/charts/aws/values.yaml
@@ -1,6 +1,4 @@
-secretName: capi-cluster-token
-sshKeyName: kubeaid-demo
-region: us-east-2
+secretName: cloud-credentials
 bastion:
   enabled: true
 vpc:
@@ -10,23 +8,3 @@ pods:
 
 controlPlane:
   loadBalancerType: nlb
-  region: us-east-2
-  replicas: 1
-  instanceType: t2.medium
-  ami:
-    id: ami-05145146e3a9db6f3
-
-machinePools:
-  - name: primary
-    replicas: 1
-    region: us-east-2
-    instanceType: t2.medium
-    sshKeyName: kubeaid-demo
-    ami:
-      id: ami-05145146e3a9db6f3
-  - name: secondary
-    replicas: 1
-    instanceType: t2.medium
-    sshKeyName: kubeaid-demo
-    ami:
-      id: ami-05145146e3a9db6f3
diff --git a/argocd-helm-charts/capi-cluster/examples/aws/cluster-autoscaler.yaml b/argocd-helm-charts/capi-cluster/examples/aws/cluster-autoscaler.yaml
@@ -0,0 +1,40 @@
+global:
+  clusterName: kubeaid-demo
+  kubeaidConfig:
+    repo: https://github.com/Archisman-Mridha/kubeaid-config
+
+provider:
+  aws: true
+
+aws:
+  secretName: cloud-credentials
+  sshKeyName: kubeaid-demo
+  region: us-east-1
+  bastion:
+    enabled: true
+  vpc:
+    cidrBlock: 10.14.0.0/22
+  pods:
+    cidrBlock: 10.244.0.0/16
+  controlPlane:
+    loadBalancer:
+      loadBalancerType: nlb
+    region: us-east-1
+    replicas: 1
+    instanceType: t4g.medium
+    ami:
+      id: ami-0fc044fa0d061d18d
+  nodeGroups:
+    - name: primary
+      # minSize: 1
+      # maxSize: 3
+      # replicas: 2
+      instanceType: t4g.medium
+      sshKeyName: kubeaid-demo
+      ami:
+        id: ami-0fc044fa0d061d18d
+      rootVolumeSize: 35
+      labels:
+        node.cluster.x-k8s.io/nodegroup: primary
+        node-role.kubernetes.io/bootstrapper: ""
+      taints: []
diff --git a/argocd-helm-charts/capi-cluster/examples/aws/machinepools.yaml b/argocd-helm-charts/capi-cluster/examples/aws/machinepools.yaml
@@ -0,0 +1,43 @@
+global:
+  clusterName: kubeaid-demo
+  enableClusterAutoscaler: false
+  capa:
+    enableMachinePools: true
+  kubeaidConfig:
+    repo: https://github.com/Archisman-Mridha/kubeaid-config
+
+provider:
+  aws: true
+
+aws:
+  secretName: cloud-credentials
+  sshKeyName: kubeaid-demo
+  region: us-east-1
+  bastion:
+    enabled: true
+  vpc:
+    cidrBlock: 10.14.0.0/22
+  pods:
+    cidrBlock: 10.244.0.0/16
+  controlPlane:
+    loadBalancer:
+      loadBalancerType: nlb
+    region: us-east-1
+    replicas: 1
+    instanceType: t4g.medium
+    ami:
+      id: ami-0fc044fa0d061d18d
+  nodeGroups:
+    - name: primary
+      # minSize: 1
+      # maxSize: 3
+      # replicas: 2
+      instanceType: t4g.medium
+      sshKeyName: kubeaid-demo
+      ami:
+        id: ami-0fc044fa0d061d18d
+      rootVolumeSize: 35
+      labels:
+        node.cluster.x-k8s.io/nodegroup: primary
+        node-role.kubernetes.io/bootstrapper: ""
+      taints: []
diff --git a/argocd-helm-charts/capi-cluster/templates/provider-aws.yaml b/argocd-helm-charts/capi-cluster/templates/provider-aws.yaml
@@ -1,10 +1,10 @@
 {{- if (.Values.provider).aws }}
 
-# For non Obmondo users, no customerid is needed.
+{{- /* For non Obmondo users, no customerid is needed. */ -}}
 {{- $name := "aws" }}
 {{- $namespace := "capi-cluster" }}
 
-# For obmondo customers.
+{{- /* For obmondo customers. */ -}}
 {{- if .Values.global.customerid }}
 {{- $name = printf "%s-%s" $name .Values.global.customerid }}
 {{- $namespace = printf "%s-%s" $namespace .Values.global.customerid }}

diff --git a/argocd-helm-charts/capi-cluster/values.yaml b/argocd-helm-charts/capi-cluster/values.yaml
@@ -3,9 +3,10 @@ global:
     version: v1.31.0
   clusterAPI:
     version: v1.8.4
+  enableClusterAutoscaler: true
   capa:
     version: v2.7.1
-    autoScaler: true
+    enableMachinePools: false
   caph:
     version: v1.0.0-beta.43
   capz: