From 87d9b018739660ebe3bb85e9388a3709097c6633 Mon Sep 17 00:00:00 2001 From: "k8s-on-aws-manager-app[bot]" <181735053+k8s-on-aws-manager-app[bot]@users.noreply.github.com> Date: Mon, 21 Oct 2024 16:33:13 +0000 Subject: [PATCH 1/5] skipper: Update to version v0.21.223 Update 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/skipper to version v0.21.223 --- cluster/node-pools/master-default/userdata.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cluster/node-pools/master-default/userdata.yaml b/cluster/node-pools/master-default/userdata.yaml index 044dfb88d8..1384d90f91 100644 --- a/cluster/node-pools/master-default/userdata.yaml +++ b/cluster/node-pools/master-default/userdata.yaml @@ -247,7 +247,7 @@ write_files: name: admission-controller-kubeconfig readOnly: true - name: skipper-admission-webhook - image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/skipper:v0.21.222 + image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/skipper:v0.21.223 args: - webhook - --address=:9085 @@ -424,7 +424,7 @@ write_files: value: {{ .Cluster.ConfigItems.apiserver_business_partner_ids }} {{ end }} - name: skipper-proxy - image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/skipper:v0.21.222 + image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/skipper:v0.21.223 args: - skipper - -access-log-strip-query @@ -475,7 +475,7 @@ write_files: name: ssl-certs-kubernetes readOnly: true - name: skipper-metrics - image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/skipper:v0.21.222 + image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/skipper:v0.21.223 args: - skipper - -access-log-strip-query From e3965937da0ec86e2d6bfcd02f159cf78e4b2041 Mon Sep 17 00:00:00 2001 From: Mahmoud Gaballah Date: Mon, 3 Jun 2024 11:57:31 +0200 Subject: [PATCH 2/5] flex karpenter node-pool configurations Signed-off-by: Mahmoud Gaballah --- cluster/config-defaults.yaml | 6 ++++ .../01-admission-control/config.yaml | 2 ++ .../worker-karpenter/provisioners.yaml | 29 +++++++++++++++---- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/cluster/config-defaults.yaml b/cluster/config-defaults.yaml index d940dda514..952b97bc17 100644 --- a/cluster/config-defaults.yaml +++ b/cluster/config-defaults.yaml @@ -1146,3 +1146,9 @@ control_plane_graceful_shutdown: "true" # fs.aio-max-nr = 8388608 # fs.inotify.max_user_watches = 100000 sysctl_settings: "" + + + +# scheduling_controls +teapot_admission_controller_scheduling_controls_enabled: "false" +teapot_admission_controller_scheduling_controls_default_architecture: "amd64" diff --git a/cluster/manifests/01-admission-control/config.yaml b/cluster/manifests/01-admission-control/config.yaml index fe203df9c2..b07b8ef4a0 100644 --- a/cluster/manifests/01-admission-control/config.yaml +++ b/cluster/manifests/01-admission-control/config.yaml @@ -52,6 +52,8 @@ data: {{- end}} pod.env-inject.node-options.enable: "{{ .Cluster.ConfigItems.teapot_admission_controller_inject_node_options_environment_variable }}" + pod.scheduling-controls.enable: "{{ .Cluster.ConfigItems.teapot_admission_controller_scheduling_controls_enabled }}" + pod.scheduling-controls.default-architecture: "{{ .Cluster.ConfigItems.teapot_admission_controller_scheduling_controls_default_architecture }}" podfactory.container-resource-check.enable: "{{ .Cluster.ConfigItems.teapot_admission_controller_validate_pod_template_resources }}" podfactory.application-label-check.enable: "{{ .Cluster.ConfigItems.teapot_admission_controller_validate_application_label }}" diff --git a/cluster/node-pools/worker-karpenter/provisioners.yaml b/cluster/node-pools/worker-karpenter/provisioners.yaml index b75565a805..9fe8f0513a 100644 --- a/cluster/node-pools/worker-karpenter/provisioners.yaml +++ b/cluster/node-pools/worker-karpenter/provisioners.yaml @@ -155,6 +155,18 @@ spec: - "c7in" - "m7in" - "r7in" +#{{ else if (gt (len .NodePool.InstanceTypes) 0) }} + - key: "node.kubernetes.io/instance-type" + operator: In + values: +# {{ range $type := .NodePool.InstanceTypes }} + - "{{ $type }}" +# {{ end }} +#{{ end }} + +# safety guards to prevent the use of unwanted instance types in case the user has not specified any specific instance types +#{{ if or (eq .NodePool.KarpenterInstanceTypeStrategy "default-for-karpenter") (eq .NodePool.KarpenterInstanceTypeStrategy "not-specified") }} + # exclude unwanted sizes - key: "karpenter.k8s.aws/instance-size" operator: "NotIn" values: @@ -166,14 +178,19 @@ spec: - "c5d.large" - "m5d.large" - "r5d.large" -#{{ else }} - - key: "node.kubernetes.io/instance-type" - operator: In +#{{end}} + +#{{ if (index .NodePool.ConfigItems "requirements") }} +# {{ range $requirement := .NodePool.KarpenterRequirements }} + - key: "{{ $requirement.Key }}" + operator: "{{ $requirement.Operator }}" values: -# {{ range $type := .NodePool.InstanceTypes }} - - "{{ $type }}" -# {{ end }} +# {{ range $value := $requirement.Values }} + - "{{ $value}}" +# {{ end }} +# {{ end }} #{{ end }} + # other configuration - key: "karpenter.sh/capacity-type" operator: In values: From 03d3dcec100881bd153603f5aa63d75240f3110b Mon Sep 17 00:00:00 2001 From: Mikkel Oscar Lyderik Larsen Date: Tue, 22 Oct 2024 10:59:50 +0200 Subject: [PATCH 3/5] Make kube-node-ready optional Signed-off-by: Mikkel Oscar Lyderik Larsen --- cluster/config-defaults.yaml | 3 +++ cluster/manifests/deletions.yaml | 11 +++++++++++ cluster/manifests/kube-node-ready/01-rbac.yaml | 2 ++ cluster/manifests/kube-node-ready/daemonset.yaml | 2 ++ cluster/manifests/kube-node-ready/service.yaml | 2 ++ cluster/node-pools/worker-combined/stack.yaml | 2 ++ cluster/node-pools/worker-splitaz/stack.yaml | 2 ++ 7 files changed, 24 insertions(+) diff --git a/cluster/config-defaults.yaml b/cluster/config-defaults.yaml index 952b97bc17..d866f55884 100644 --- a/cluster/config-defaults.yaml +++ b/cluster/config-defaults.yaml @@ -484,6 +484,9 @@ kubernetes_lifecycle_metrics_mem_min: "120Mi" kube_node_ready_controller_cpu: "50m" kube_node_ready_controller_memory: "200Mi" +# Enable kube-node-ready ASG lifecycle hook feature. +kube_node_ready_enabled: "true" + # Enable deployment of aws-cloud-controller-manager aws_cloud_controller_manager_enabled: "true" aws_cloud_controller_manager_cpu: "125m" diff --git a/cluster/manifests/deletions.yaml b/cluster/manifests/deletions.yaml index eaad055f02..401057c9a8 100644 --- a/cluster/manifests/deletions.yaml +++ b/cluster/manifests/deletions.yaml @@ -309,3 +309,14 @@ post_apply: kind: DaemonSet namespace: kube-system {{- end }} +{{- if ne .Cluster.ConfigItems.kube_node_ready_enabled "true" }} +- name: kube-node-ready + kind: DaemonSet + namespace: kube-system +- name: kube-node-ready + kind: ServiceAccount + namespace: kube-system +- name: kube-node-ready + kind: Service + namespace: kube-system +{{- end }} diff --git a/cluster/manifests/kube-node-ready/01-rbac.yaml b/cluster/manifests/kube-node-ready/01-rbac.yaml index e38976084d..e6d2d09cab 100644 --- a/cluster/manifests/kube-node-ready/01-rbac.yaml +++ b/cluster/manifests/kube-node-ready/01-rbac.yaml @@ -1,3 +1,4 @@ +# {{ if eq .Cluster.ConfigItems.kube_node_ready_enabled "true" }} apiVersion: v1 kind: ServiceAccount metadata: @@ -5,3 +6,4 @@ metadata: namespace: kube-system annotations: iam.amazonaws.com/role: "{{ .Cluster.LocalID }}-kube-node-ready" +# {{ end }} diff --git a/cluster/manifests/kube-node-ready/daemonset.yaml b/cluster/manifests/kube-node-ready/daemonset.yaml index fc7594203b..65f2e4d12b 100644 --- a/cluster/manifests/kube-node-ready/daemonset.yaml +++ b/cluster/manifests/kube-node-ready/daemonset.yaml @@ -1,3 +1,4 @@ +# {{ if eq .Cluster.ConfigItems.kube_node_ready_enabled "true" }} # {{ $image := "container-registry.zalando.net/teapot/kube-node-ready:master-34" }} # {{ $version := index (split $image ":") 1 }} @@ -65,3 +66,4 @@ spec: runAsUser: 1000 securityContext: fsGroup: 65534 +# {{ end }} diff --git a/cluster/manifests/kube-node-ready/service.yaml b/cluster/manifests/kube-node-ready/service.yaml index 27681e9759..5a9fb3a838 100644 --- a/cluster/manifests/kube-node-ready/service.yaml +++ b/cluster/manifests/kube-node-ready/service.yaml @@ -1,3 +1,4 @@ +# {{ if eq .Cluster.ConfigItems.kube_node_ready_enabled "true" }} kind: Service apiVersion: v1 metadata: @@ -16,3 +17,4 @@ spec: protocol: TCP selector: component: kube-node-ready +# {{ end }} diff --git a/cluster/node-pools/worker-combined/stack.yaml b/cluster/node-pools/worker-combined/stack.yaml index 632dbaf667..38a0d1e285 100644 --- a/cluster/node-pools/worker-combined/stack.yaml +++ b/cluster/node-pools/worker-combined/stack.yaml @@ -174,6 +174,7 @@ Resources: Roles: - !ImportValue '{{ .Cluster.ID }}:worker-iam-role' Type: 'AWS::IAM::InstanceProfile' +# {{ if eq .Cluster.ConfigItems.kube_node_ready_enabled "true" }} AutoscalingLifecycleHook: Properties: AutoScalingGroupName: !Ref AutoScalingGroup @@ -182,3 +183,4 @@ Resources: HeartbeatTimeout: '600' LifecycleTransition: 'autoscaling:EC2_INSTANCE_LAUNCHING' Type: 'AWS::AutoScaling::LifecycleHook' +# {{ end }} diff --git a/cluster/node-pools/worker-splitaz/stack.yaml b/cluster/node-pools/worker-splitaz/stack.yaml index 43a20bc1ce..10dafb3811 100644 --- a/cluster/node-pools/worker-splitaz/stack.yaml +++ b/cluster/node-pools/worker-splitaz/stack.yaml @@ -125,6 +125,7 @@ Resources: VPCZoneIdentifier: - "{{ index $data.Values.subnets $az }}" Type: 'AWS::AutoScaling::AutoScalingGroup' +# {{ if eq $data.Cluster.ConfigItems.kube_node_ready_enabled "true" }} AutoscalingLifecycleHook{{$azID}}: Properties: AutoScalingGroupName: !Ref AutoScalingGroup{{$azID}} @@ -133,6 +134,7 @@ Resources: HeartbeatTimeout: '600' LifecycleTransition: 'autoscaling:EC2_INSTANCE_LAUNCHING' Type: 'AWS::AutoScaling::LifecycleHook' +# {{ end }} {{ end }} {{ end }} {{ end }} From 075e7cc716d78e37c82ccc801fc515bcf83cb7c3 Mon Sep 17 00:00:00 2001 From: "k8s-on-aws-manager-app[bot]" <181735053+k8s-on-aws-manager-app[bot]@users.noreply.github.com> Date: Tue, 22 Oct 2024 10:02:29 +0000 Subject: [PATCH 4/5] kube2iam: Update to version 0.12.0-master-19.patched Update container-registry.zalando.net/teapot/kube2iam to version 0.12.0-master-19.patched --- cluster/manifests/kube2iam/daemonset.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/manifests/kube2iam/daemonset.yaml b/cluster/manifests/kube2iam/daemonset.yaml index 0063770dde..95426ac972 100644 --- a/cluster/manifests/kube2iam/daemonset.yaml +++ b/cluster/manifests/kube2iam/daemonset.yaml @@ -38,7 +38,7 @@ spec: effect: NoExecute hostNetwork: true containers: - - image: container-registry.zalando.net/teapot/kube2iam:0.11.2-master-18.patched + - image: container-registry.zalando.net/teapot/kube2iam:0.12.0-master-19.patched name: kube2iam args: - --auto-discover-base-arn From dddc2a3face1d6e9a691d98b270438f7438457ba Mon Sep 17 00:00:00 2001 From: Mikkel Oscar Lyderik Larsen Date: Tue, 22 Oct 2024 15:57:30 +0200 Subject: [PATCH 5/5] Set AWS_DEFAULT_REGION for kube2iam Signed-off-by: Mikkel Oscar Lyderik Larsen --- cluster/manifests/kube2iam/daemonset.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cluster/manifests/kube2iam/daemonset.yaml b/cluster/manifests/kube2iam/daemonset.yaml index 95426ac972..9ac5efd6ea 100644 --- a/cluster/manifests/kube2iam/daemonset.yaml +++ b/cluster/manifests/kube2iam/daemonset.yaml @@ -45,6 +45,8 @@ spec: - --verbose - --node=$(NODE_NAME) env: + - name: AWS_DEFAULT_REGION + value: "{{.Cluster.Region}}" - name: NODE_NAME valueFrom: fieldRef: