From d3d22c86d7f8b43bf2f19cb0c17279400a4b67ee Mon Sep 17 00:00:00 2001 From: fullykubed Date: Fri, 29 Mar 2024 08:18:06 -0400 Subject: [PATCH] fix: fixes node memory reservations --- packages/terraform/aws_eks/main.tf | 1 + packages/terraform/kube_cilium/main.tf | 4 ++-- packages/terraform/kube_ingress_nginx/main.tf | 2 +- .../terraform/kube_karpenter_node_pools/main.tf | 16 +++++++++++++++- packages/terraform/kube_node_settings/main.tf | 4 ++-- packages/terraform/kube_node_settings/output.tf | 2 +- .../terraform/kube_node_settings/user-data.toml | 3 +++ packages/terraform/kube_node_settings/vars.tf | 6 ++++++ .../kube_karpenter_node_pools/page.mdx | 4 ++++ .../kube_node_settings/page.mdx | 8 ++++++++ 10 files changed, 43 insertions(+), 7 deletions(-) diff --git a/packages/terraform/aws_eks/main.tf b/packages/terraform/aws_eks/main.tf index 16b4c2bb..3ebc3532 100644 --- a/packages/terraform/aws_eks/main.tf +++ b/packages/terraform/aws_eks/main.tf @@ -45,6 +45,7 @@ module "node_settings" { cluster_name = aws_eks_cluster.cluster.name cluster_ca_data = aws_eks_cluster.cluster.certificate_authority[0].data cluster_endpoint = aws_eks_cluster.cluster.endpoint + max_pods = 25 environment = var.environment pf_root_module = var.pf_root_module region = var.region diff --git a/packages/terraform/kube_cilium/main.tf b/packages/terraform/kube_cilium/main.tf index d2ca3f86..844c6d5a 100644 --- a/packages/terraform/kube_cilium/main.tf +++ b/packages/terraform/kube_cilium/main.tf @@ -209,7 +209,7 @@ resource "helm_release" "cilium" { memory = "100Mi" } limits = { - memory = "200Mi" + memory = "130Mi" } } @@ -242,7 +242,7 @@ resource "helm_release" "cilium" { memory = "100Mi" } limits = { - memory = "150Mi" + memory = "130Mi" } } diff --git a/packages/terraform/kube_ingress_nginx/main.tf b/packages/terraform/kube_ingress_nginx/main.tf index 151ccfb0..741d72dc 100644 --- a/packages/terraform/kube_ingress_nginx/main.tf +++ b/packages/terraform/kube_ingress_nginx/main.tf @@ -323,7 +323,7 @@ resource "helm_release" "nginx_ingress" { } } minReadySeconds = 10 - minAvailable = "67%" + maxUnavailable = 1 tolerations = module.constants.burstable_node_toleration_helm affinity = module.constants.pod_anti_affinity_helm diff --git a/packages/terraform/kube_karpenter_node_pools/main.tf b/packages/terraform/kube_karpenter_node_pools/main.tf index 99573674..a12ca79f 100644 --- a/packages/terraform/kube_karpenter_node_pools/main.tf +++ b/packages/terraform/kube_karpenter_node_pools/main.tf @@ -83,11 +83,25 @@ module "constants" { extra_tags = var.extra_tags } +module "node_settings_burstable" { + source = "../kube_node_settings" + cluster_name = var.cluster_name + cluster_endpoint = var.cluster_endpoint + cluster_ca_data = var.cluster_ca_data + max_pods = 20 + environment = var.environment + pf_root_module = var.pf_root_module + region = var.region + is_local = var.is_local + extra_tags = var.extra_tags +} + module "node_settings" { source = "../kube_node_settings" cluster_name = var.cluster_name cluster_endpoint = var.cluster_endpoint cluster_ca_data = var.cluster_ca_data + max_pods = 40 environment = var.environment pf_root_module = var.pf_root_module region = var.region @@ -131,7 +145,7 @@ resource "kubernetes_manifest" "default_node_class" { httpPutResponseHopLimit = 1 // don't allow pods to access the node roles httpTokens = "required" } - userData = module.node_settings.user_data + userData = module.node_settings_burstable.user_data blockDeviceMappings = [ { deviceName = "/dev/xvda" diff --git a/packages/terraform/kube_node_settings/main.tf b/packages/terraform/kube_node_settings/main.tf index b42d1bca..099d1a7e 100644 --- a/packages/terraform/kube_node_settings/main.tf +++ b/packages/terraform/kube_node_settings/main.tf @@ -10,13 +10,13 @@ locals { eviction_soft_grace_period_inodes_free = "2m0s" image_gc_high_threshold_percent = "85" image_gc_low_threshold_percent = "80" - max_pods = "100" shutdown_grace_period = "60m0s" user_data = templatefile("${path.module}/user-data.toml", { API_SERVER_ADDR = var.cluster_endpoint CLUSTER_CA_DATA = var.cluster_ca_data CLUSTER_NAME = var.cluster_name - MAX_PODS = local.max_pods + MAX_PODS = var.max_pods + KUBE_RESERVED_MEMORY = 150 + (6 * var.max_pods) SHUTDOWN_GRACE_PERIOD = local.shutdown_grace_period IMAGE_GC_HIGH_THRESHOLD_PERCENT = local.image_gc_high_threshold_percent IMAGE_GC_LOW_THRESHOLD_PERCENT = local.image_gc_low_threshold_percent diff --git a/packages/terraform/kube_node_settings/output.tf b/packages/terraform/kube_node_settings/output.tf index 41fb34a8..e5aebf4b 100644 --- a/packages/terraform/kube_node_settings/output.tf +++ b/packages/terraform/kube_node_settings/output.tf @@ -32,7 +32,7 @@ output "image_gc_low_threshold_percent" { value = local.image_gc_low_threshold_percent } output "max_pods" { - value = local.max_pods + value = var.max_pods } output "shutdown_grace_period" { value = local.shutdown_grace_period diff --git a/packages/terraform/kube_node_settings/user-data.toml b/packages/terraform/kube_node_settings/user-data.toml index 5962d41f..cc4b67e0 100644 --- a/packages/terraform/kube_node_settings/user-data.toml +++ b/packages/terraform/kube_node_settings/user-data.toml @@ -19,3 +19,6 @@ image-gc-low-threshold-percent = "${IMAGE_GC_LOW_THRESHOLD_PERCENT}" "memory.available" = "${EVICTION_SOFT_GRACE_PERIOD_MEMORY_AVAILABLE}" "nodefs.available" = "${EVICTION_SOFT_GRACE_PERIOD_NODEFS_AVAILABLE}" "nodefs.inodesFree" = "${EVICTION_SOFT_GRACE_PERIOD_INODES_FREE}" + +[settings.kubernetes.kube-reserved] +memory = "${KUBE_RESERVED_MEMORY}Mi" diff --git a/packages/terraform/kube_node_settings/vars.tf b/packages/terraform/kube_node_settings/vars.tf index 13b32bd8..d5dc7b18 100644 --- a/packages/terraform/kube_node_settings/vars.tf +++ b/packages/terraform/kube_node_settings/vars.tf @@ -15,3 +15,9 @@ variable "cluster_name" { type = string default = "" } + +variable "max_pods" { + description = "The maximum number of pods to run on this node type" + type = number + default = 50 +} diff --git a/packages/website/src/app/(web)/docs/reference/terraform-modules/kube_karpenter_node_pools/page.mdx b/packages/website/src/app/(web)/docs/reference/terraform-modules/kube_karpenter_node_pools/page.mdx index 2903e6c1..0106b851 100644 --- a/packages/website/src/app/(web)/docs/reference/terraform-modules/kube_karpenter_node_pools/page.mdx +++ b/packages/website/src/app/(web)/docs/reference/terraform-modules/kube_karpenter_node_pools/page.mdx @@ -26,6 +26,10 @@ Source: [kube\_labels](./kube_labels) Source: [kube\_node\_settings](./kube_node_settings) +### [node\_settings\_burstable](#module_node_settings_burstable) + +Source: [kube\_node\_settings](./kube_node_settings) + ## Required Inputs The following input variables are required: diff --git a/packages/website/src/app/(web)/docs/reference/terraform-modules/kube_node_settings/page.mdx b/packages/website/src/app/(web)/docs/reference/terraform-modules/kube_node_settings/page.mdx index 14888872..57e9ba8d 100644 --- a/packages/website/src/app/(web)/docs/reference/terraform-modules/kube_node_settings/page.mdx +++ b/packages/website/src/app/(web)/docs/reference/terraform-modules/kube_node_settings/page.mdx @@ -76,6 +76,14 @@ Type: `bool` Default: `false` +### [max\_pods](#input_max_pods) + +Description: The maximum number of pods to run on this node type + +Type: `number` + +Default: `50` + ### [pf\_module](#input_pf_module) Description: The name of the Panfactum module where the containing resources are directly defined.