fix: fixes node memory reservations

Panfactum · Mar 29, 2024 · d3d22c8 · d3d22c8
1 parent a599e58
commit d3d22c8
Show file tree

Hide file tree

Showing 10 changed files with 43 additions and 7 deletions.
diff --git a/packages/terraform/aws_eks/main.tf b/packages/terraform/aws_eks/main.tf
@@ -45,6 +45,7 @@ module "node_settings" {
   cluster_name     = aws_eks_cluster.cluster.name
   cluster_ca_data  = aws_eks_cluster.cluster.certificate_authority[0].data
   cluster_endpoint = aws_eks_cluster.cluster.endpoint
+  max_pods         = 25
   environment      = var.environment
   pf_root_module   = var.pf_root_module
   region           = var.region

diff --git a/packages/terraform/kube_cilium/main.tf b/packages/terraform/kube_cilium/main.tf
@@ -209,7 +209,7 @@ resource "helm_release" "cilium" {
           memory = "100Mi"
         }
         limits = {
-          memory = "200Mi"
+          memory = "130Mi"
         }
       }
 
@@ -242,7 +242,7 @@ resource "helm_release" "cilium" {
             memory = "100Mi"
           }
           limits = {
-            memory = "150Mi"
+            memory = "130Mi"
           }
         }
 

diff --git a/packages/terraform/kube_ingress_nginx/main.tf b/packages/terraform/kube_ingress_nginx/main.tf
@@ -323,7 +323,7 @@ resource "helm_release" "nginx_ingress" {
           }
         }
         minReadySeconds = 10
-        minAvailable    = "67%"
+        maxUnavailable  = 1
 
         tolerations               = module.constants.burstable_node_toleration_helm
         affinity                  = module.constants.pod_anti_affinity_helm

diff --git a/packages/terraform/kube_karpenter_node_pools/main.tf b/packages/terraform/kube_karpenter_node_pools/main.tf
@@ -83,11 +83,25 @@ module "constants" {
   extra_tags     = var.extra_tags
 }
 
+module "node_settings_burstable" {
+  source           = "../kube_node_settings"
+  cluster_name     = var.cluster_name
+  cluster_endpoint = var.cluster_endpoint
+  cluster_ca_data  = var.cluster_ca_data
+  max_pods         = 20
+  environment      = var.environment
+  pf_root_module   = var.pf_root_module
+  region           = var.region
+  is_local         = var.is_local
+  extra_tags       = var.extra_tags
+}
+
 module "node_settings" {
   source           = "../kube_node_settings"
   cluster_name     = var.cluster_name
   cluster_endpoint = var.cluster_endpoint
   cluster_ca_data  = var.cluster_ca_data
+  max_pods         = 40
   environment      = var.environment
   pf_root_module   = var.pf_root_module
   region           = var.region
@@ -131,7 +145,7 @@ resource "kubernetes_manifest" "default_node_class" {
         httpPutResponseHopLimit = 1 // don't allow pods to access the node roles
         httpTokens              = "required"
       }
-      userData = module.node_settings.user_data
+      userData = module.node_settings_burstable.user_data
       blockDeviceMappings = [
         {
           deviceName = "/dev/xvda"

diff --git a/packages/terraform/kube_node_settings/main.tf b/packages/terraform/kube_node_settings/main.tf
@@ -10,13 +10,13 @@ locals {
   eviction_soft_grace_period_inodes_free      = "2m0s"
   image_gc_high_threshold_percent             = "85"
   image_gc_low_threshold_percent              = "80"
-  max_pods                                    = "100"
   shutdown_grace_period                       = "60m0s"
   user_data = templatefile("${path.module}/user-data.toml", {
     API_SERVER_ADDR                             = var.cluster_endpoint
     CLUSTER_CA_DATA                             = var.cluster_ca_data
     CLUSTER_NAME                                = var.cluster_name
-    MAX_PODS                                    = local.max_pods
+    MAX_PODS                                    = var.max_pods
+    KUBE_RESERVED_MEMORY                        = 150 + (6 * var.max_pods)
     SHUTDOWN_GRACE_PERIOD                       = local.shutdown_grace_period
     IMAGE_GC_HIGH_THRESHOLD_PERCENT             = local.image_gc_high_threshold_percent
     IMAGE_GC_LOW_THRESHOLD_PERCENT              = local.image_gc_low_threshold_percent

diff --git a/packages/terraform/kube_node_settings/output.tf b/packages/terraform/kube_node_settings/output.tf
@@ -32,7 +32,7 @@ output "image_gc_low_threshold_percent" {
   value = local.image_gc_low_threshold_percent
 }
 output "max_pods" {
-  value = local.max_pods
+  value = var.max_pods
 }
 output "shutdown_grace_period" {
   value = local.shutdown_grace_period

diff --git a/packages/terraform/kube_node_settings/user-data.toml b/packages/terraform/kube_node_settings/user-data.toml
@@ -19,3 +19,6 @@ image-gc-low-threshold-percent = "${IMAGE_GC_LOW_THRESHOLD_PERCENT}"
 "memory.available" = "${EVICTION_SOFT_GRACE_PERIOD_MEMORY_AVAILABLE}"
 "nodefs.available" = "${EVICTION_SOFT_GRACE_PERIOD_NODEFS_AVAILABLE}"
 "nodefs.inodesFree" = "${EVICTION_SOFT_GRACE_PERIOD_INODES_FREE}"
+
+[settings.kubernetes.kube-reserved]
+memory = "${KUBE_RESERVED_MEMORY}Mi"
diff --git a/packages/terraform/kube_node_settings/vars.tf b/packages/terraform/kube_node_settings/vars.tf
@@ -15,3 +15,9 @@ variable "cluster_name" {
   type        = string
   default     = ""
 }
+
+variable "max_pods" {
+  description = "The maximum number of pods to run on this node type"
+  type        = number
+  default     = 50
+}
diff --git a/...c/app/(web)/docs/reference/terraform-modules/kube_karpenter_node_pools/page.mdx b/...c/app/(web)/docs/reference/terraform-modules/kube_karpenter_node_pools/page.mdx
@@ -26,6 +26,10 @@ Source: [kube\_labels](./kube_labels)
 
 Source: [kube\_node\_settings](./kube_node_settings)
 
+### <a name="module_node_settings_burstable" /> [node\_settings\_burstable](#module_node_settings_burstable)
+
+Source: [kube\_node\_settings](./kube_node_settings)
+
 ## Required Inputs
 
 The following input variables are required:

diff --git a/...site/src/app/(web)/docs/reference/terraform-modules/kube_node_settings/page.mdx b/...site/src/app/(web)/docs/reference/terraform-modules/kube_node_settings/page.mdx
@@ -76,6 +76,14 @@ Type: `bool`
 
 Default: `false`
 
+### <a name="input_max_pods" /> [max\_pods](#input_max_pods)
+
+Description: The maximum number of pods to run on this node type
+
+Type: `number`
+
+Default: `50`
+
 ### <a name="input_pf_module" /> [pf\_module](#input_pf_module)
 
 Description: The name of the Panfactum module where the containing resources are directly defined.