Merge pull request #8429 from zalando-incubator/dev-to-alpha

dev to alpha

cluster/config-defaults.yaml

@@ -484,6 +484,9 @@ kubernetes_lifecycle_metrics_mem_min: "120Mi"
 kube_node_ready_controller_cpu: "50m"
 kube_node_ready_controller_memory: "200Mi"
 
+# Enable kube-node-ready ASG lifecycle hook feature.
+kube_node_ready_enabled: "true"
+
 # Enable deployment of aws-cloud-controller-manager
 aws_cloud_controller_manager_enabled: "true"
 aws_cloud_controller_manager_cpu: "125m"
@@ -1146,3 +1149,9 @@ control_plane_graceful_shutdown: "true"
 #  fs.aio-max-nr = 8388608
 #  fs.inotify.max_user_watches = 100000
 sysctl_settings: ""
+
+
+
+# scheduling_controls
+teapot_admission_controller_scheduling_controls_enabled: "false"
+teapot_admission_controller_scheduling_controls_default_architecture: "amd64"

cluster/manifests/01-admission-control/config.yaml

@@ -52,6 +52,8 @@ data:
 {{- end}}
 
   pod.env-inject.node-options.enable: "{{ .Cluster.ConfigItems.teapot_admission_controller_inject_node_options_environment_variable }}"
+  pod.scheduling-controls.enable: "{{ .Cluster.ConfigItems.teapot_admission_controller_scheduling_controls_enabled }}"
+  pod.scheduling-controls.default-architecture: "{{ .Cluster.ConfigItems.teapot_admission_controller_scheduling_controls_default_architecture }}"
 
   podfactory.container-resource-check.enable: "{{ .Cluster.ConfigItems.teapot_admission_controller_validate_pod_template_resources }}"
   podfactory.application-label-check.enable: "{{ .Cluster.ConfigItems.teapot_admission_controller_validate_application_label }}"

cluster/manifests/deletions.yaml

@@ -309,3 +309,14 @@ post_apply:
   kind: DaemonSet
   namespace: kube-system
 {{- end }}
+{{- if ne .Cluster.ConfigItems.kube_node_ready_enabled "true" }}
+- name: kube-node-ready
+  kind: DaemonSet
+  namespace: kube-system
+- name: kube-node-ready
+  kind: ServiceAccount
+  namespace: kube-system
+- name: kube-node-ready
+  kind: Service
+  namespace: kube-system
+{{- end }}

cluster/manifests/kube-node-ready/01-rbac.yaml

@@ -1,7 +1,9 @@
+# {{ if eq .Cluster.ConfigItems.kube_node_ready_enabled "true" }}
 apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: kube-node-ready
   namespace: kube-system
   annotations:
     iam.amazonaws.com/role: "{{ .Cluster.LocalID }}-kube-node-ready"
+# {{ end }}

cluster/manifests/kube-node-ready/daemonset.yaml

@@ -1,3 +1,4 @@
+# {{ if eq .Cluster.ConfigItems.kube_node_ready_enabled "true" }}
 # {{ $image := "container-registry.zalando.net/teapot/kube-node-ready:master-34" }}
 # {{ $version := index (split $image ":") 1 }}
 
@@ -65,3 +66,4 @@ spec:
           runAsUser: 1000
       securityContext:
         fsGroup: 65534
+# {{ end }}

cluster/manifests/kube-node-ready/service.yaml

@@ -1,3 +1,4 @@
+# {{ if eq .Cluster.ConfigItems.kube_node_ready_enabled "true" }}
 kind: Service
 apiVersion: v1
 metadata:
@@ -16,3 +17,4 @@ spec:
       protocol: TCP
   selector:
     component: kube-node-ready
+# {{ end }}

cluster/manifests/kube2iam/daemonset.yaml

@@ -38,13 +38,15 @@ spec:
         effect: NoExecute
       hostNetwork: true
       containers:
-      - image: container-registry.zalando.net/teapot/kube2iam:0.11.2-master-18.patched
+      - image: container-registry.zalando.net/teapot/kube2iam:0.12.0-master-19.patched
         name: kube2iam
         args:
         - --auto-discover-base-arn
         - --verbose
         - --node=$(NODE_NAME)
         env:
+        - name: AWS_DEFAULT_REGION
+          value: "{{.Cluster.Region}}"
         - name: NODE_NAME
           valueFrom:
             fieldRef:

cluster/node-pools/master-default/userdata.yaml

@@ -247,7 +247,7 @@ write_files:
               name: admission-controller-kubeconfig
               readOnly: true
         - name: skipper-admission-webhook
-          image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/skipper:v0.21.222
+          image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/skipper:v0.21.223
           args:
             - webhook
             - --address=:9085
@@ -424,7 +424,7 @@ write_files:
             value: {{ .Cluster.ConfigItems.apiserver_business_partner_ids }}
 {{ end }}
         - name: skipper-proxy
-          image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/skipper:v0.21.222
+          image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/skipper:v0.21.223
           args:
           - skipper
           - -access-log-strip-query
@@ -475,7 +475,7 @@ write_files:
             name: ssl-certs-kubernetes
             readOnly: true
         - name: skipper-metrics
-          image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/skipper:v0.21.222
+          image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/skipper:v0.21.223
           args:
           - skipper
           - -access-log-strip-query

cluster/node-pools/worker-combined/stack.yaml

@@ -174,6 +174,7 @@ Resources:
       Roles:
       - !ImportValue '{{ .Cluster.ID }}:worker-iam-role'
     Type: 'AWS::IAM::InstanceProfile'
+# {{ if eq .Cluster.ConfigItems.kube_node_ready_enabled "true" }}
   AutoscalingLifecycleHook:
     Properties:
       AutoScalingGroupName: !Ref AutoScalingGroup
@@ -182,3 +183,4 @@ Resources:
       HeartbeatTimeout: '600'
       LifecycleTransition: 'autoscaling:EC2_INSTANCE_LAUNCHING'
     Type: 'AWS::AutoScaling::LifecycleHook'
+# {{ end }}

cluster/node-pools/worker-karpenter/provisioners.yaml

@@ -155,6 +155,18 @@ spec:
             - "c7in"
             - "m7in"
             - "r7in"
+#{{ else if (gt (len .NodePool.InstanceTypes) 0) }}
+        - key: "node.kubernetes.io/instance-type"
+          operator: In
+          values:
+#      {{ range $type := .NodePool.InstanceTypes }}
+          - "{{ $type }}"
+#      {{ end }}
+#{{ end }}
+
+# safety guards to prevent the use of unwanted instance types in case the user has not specified any specific instance types
+#{{ if or (eq .NodePool.KarpenterInstanceTypeStrategy "default-for-karpenter") (eq .NodePool.KarpenterInstanceTypeStrategy "not-specified") }}
+        # exclude unwanted sizes
         - key: "karpenter.k8s.aws/instance-size"
           operator: "NotIn"
           values:
@@ -166,14 +178,19 @@ spec:
             - "c5d.large"
             - "m5d.large"
             - "r5d.large"
-#{{ else }}
-        - key: "node.kubernetes.io/instance-type"
-          operator: In
+#{{end}}
+
+#{{ if (index .NodePool.ConfigItems "requirements") }}
+#    {{ range $requirement := .NodePool.KarpenterRequirements }}
+        - key: "{{ $requirement.Key }}"
+          operator: "{{ $requirement.Operator }}"
           values:
-#      {{ range $type := .NodePool.InstanceTypes }}
-          - "{{ $type }}"
-#      {{ end }}
+#         {{ range $value := $requirement.Values }}
+            - "{{ $value}}"
+#         {{ end }}
+#    {{ end }}
 #{{ end }}
+        # other configuration
         - key: "karpenter.sh/capacity-type"
           operator: In
           values:

cluster/node-pools/worker-splitaz/stack.yaml

@@ -125,6 +125,7 @@ Resources:
       VPCZoneIdentifier:
         - "{{ index $data.Values.subnets $az }}"
     Type: 'AWS::AutoScaling::AutoScalingGroup'
+# {{ if eq $data.Cluster.ConfigItems.kube_node_ready_enabled "true" }}
   AutoscalingLifecycleHook{{$azID}}:
     Properties:
       AutoScalingGroupName: !Ref AutoScalingGroup{{$azID}}
@@ -133,6 +134,7 @@ Resources:
       HeartbeatTimeout: '600'
       LifecycleTransition: 'autoscaling:EC2_INSTANCE_LAUNCHING'
     Type: 'AWS::AutoScaling::LifecycleHook'
+# {{ end }}
 {{ end }}
 {{ end }}
 {{ end }}