Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add node graceful shutdown kubelet config #4149

Closed
wants to merge 5 commits into from
Closed

feat: add node graceful shutdown kubelet config #4149

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
23e1964
Add shutdown config
weichch Feb 20, 2024
50e45e5
Convert to kubelet config
weichch Feb 20, 2024
2864425
Add baker test
weichch Feb 20, 2024
e0d0e87
Merge branch 'master' into weichen/graceful-node-shutdown
cameronmeissner Mar 12, 2024
6c10649
Merge branch 'master' into weichen/graceful-node-shutdown
weichch Mar 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 38 additions & 32 deletions pkg/agent/baker_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -470,17 +470,19 @@ var _ = Describe("Assert generated customData and cseCmd", func() {
var swapFileSizeMB int32 = 1500
var netCoreSomaxconn int32 = 1638499
config.ContainerService.Properties.AgentPoolProfiles[0].CustomKubeletConfig = &datamodel.CustomKubeletConfig{
CPUManagerPolicy: "static",
CPUCfsQuota: to.BoolPtr(false),
CPUCfsQuotaPeriod: "200ms",
ImageGcHighThreshold: to.Int32Ptr(90),
ImageGcLowThreshold: to.Int32Ptr(70),
TopologyManagerPolicy: "best-effort",
AllowedUnsafeSysctls: &[]string{"kernel.msg*", "net.ipv4.route.min_pmtu"},
FailSwapOn: &failSwapOn,
ContainerLogMaxSizeMB: to.Int32Ptr(1000),
ContainerLogMaxFiles: to.Int32Ptr(99),
PodMaxPids: to.Int32Ptr(12345),
CPUManagerPolicy: "static",
CPUCfsQuota: to.BoolPtr(false),
CPUCfsQuotaPeriod: "200ms",
ImageGcHighThreshold: to.Int32Ptr(90),
ImageGcLowThreshold: to.Int32Ptr(70),
TopologyManagerPolicy: "best-effort",
AllowedUnsafeSysctls: &[]string{"kernel.msg*", "net.ipv4.route.min_pmtu"},
FailSwapOn: &failSwapOn,
ContainerLogMaxSizeMB: to.Int32Ptr(1000),
ContainerLogMaxFiles: to.Int32Ptr(99),
PodMaxPids: to.Int32Ptr(12345),
ShutdownGracePeriod: to.Int32Ptr(30),
ShutdownGracePeriodCriticalPods: to.Int32Ptr(15),
}
config.ContainerService.Properties.AgentPoolProfiles[0].CustomLinuxOSConfig = &datamodel.CustomLinuxOSConfig{
Sysctls: &datamodel.SysctlConfig{
Expand Down Expand Up @@ -512,16 +514,18 @@ var _ = Describe("Assert generated customData and cseCmd", func() {
Entry("AKSUbuntu1604 - dynamic-config-dir should always be removed with custom kubelet config",
"AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig", "1.16.13", func(config *datamodel.NodeBootstrappingConfiguration) {
config.ContainerService.Properties.AgentPoolProfiles[0].CustomKubeletConfig = &datamodel.CustomKubeletConfig{
CPUManagerPolicy: "static",
CPUCfsQuota: to.BoolPtr(false),
CPUCfsQuotaPeriod: "200ms",
ImageGcHighThreshold: to.Int32Ptr(90),
ImageGcLowThreshold: to.Int32Ptr(70),
TopologyManagerPolicy: "best-effort",
AllowedUnsafeSysctls: &[]string{"kernel.msg*", "net.ipv4.route.min_pmtu"},
ContainerLogMaxSizeMB: to.Int32Ptr(1000),
ContainerLogMaxFiles: to.Int32Ptr(99),
PodMaxPids: to.Int32Ptr(12345),
CPUManagerPolicy: "static",
CPUCfsQuota: to.BoolPtr(false),
CPUCfsQuotaPeriod: "200ms",
ImageGcHighThreshold: to.Int32Ptr(90),
ImageGcLowThreshold: to.Int32Ptr(70),
TopologyManagerPolicy: "best-effort",
AllowedUnsafeSysctls: &[]string{"kernel.msg*", "net.ipv4.route.min_pmtu"},
ContainerLogMaxSizeMB: to.Int32Ptr(1000),
ContainerLogMaxFiles: to.Int32Ptr(99),
PodMaxPids: to.Int32Ptr(12345),
ShutdownGracePeriod: to.Int32Ptr(30),
ShutdownGracePeriodCriticalPods: to.Int32Ptr(15),
}
config.KubeletConfig = map[string]string{
"--address": "0.0.0.0",
Expand Down Expand Up @@ -1222,17 +1226,19 @@ oom_score = 0
var swapFileSizeMB int32 = 1500
var netCoreSomaxconn int32 = 1638499
config.ContainerService.Properties.AgentPoolProfiles[0].CustomKubeletConfig = &datamodel.CustomKubeletConfig{
CPUManagerPolicy: "static",
CPUCfsQuota: to.BoolPtr(false),
CPUCfsQuotaPeriod: "200ms",
ImageGcHighThreshold: to.Int32Ptr(90),
ImageGcLowThreshold: to.Int32Ptr(70),
TopologyManagerPolicy: "best-effort",
AllowedUnsafeSysctls: &[]string{"kernel.msg*", "net.ipv4.route.min_pmtu"},
FailSwapOn: &failSwapOn,
ContainerLogMaxSizeMB: to.Int32Ptr(1000),
ContainerLogMaxFiles: to.Int32Ptr(99),
PodMaxPids: to.Int32Ptr(12345),
CPUManagerPolicy: "static",
CPUCfsQuota: to.BoolPtr(false),
CPUCfsQuotaPeriod: "200ms",
ImageGcHighThreshold: to.Int32Ptr(90),
ImageGcLowThreshold: to.Int32Ptr(70),
TopologyManagerPolicy: "best-effort",
AllowedUnsafeSysctls: &[]string{"kernel.msg*", "net.ipv4.route.min_pmtu"},
FailSwapOn: &failSwapOn,
ContainerLogMaxSizeMB: to.Int32Ptr(1000),
ContainerLogMaxFiles: to.Int32Ptr(99),
PodMaxPids: to.Int32Ptr(12345),
ShutdownGracePeriod: to.Int32Ptr(30),
ShutdownGracePeriodCriticalPods: to.Int32Ptr(15),
}
config.ContainerService.Properties.AgentPoolProfiles[0].CustomLinuxOSConfig = &datamodel.CustomLinuxOSConfig{
Sysctls: &datamodel.SysctlConfig{
Expand Down
40 changes: 29 additions & 11 deletions pkg/agent/datamodel/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -669,17 +669,19 @@ type ProvisioningState string

// CustomKubeletConfig represents custom kubelet configurations for agent pool nodes.
type CustomKubeletConfig struct {
CPUManagerPolicy string `json:"cpuManagerPolicy,omitempty"`
CPUCfsQuota *bool `json:"cpuCfsQuota,omitempty"`
CPUCfsQuotaPeriod string `json:"cpuCfsQuotaPeriod,omitempty"`
ImageGcHighThreshold *int32 `json:"imageGcHighThreshold,omitempty"`
ImageGcLowThreshold *int32 `json:"imageGcLowThreshold,omitempty"`
TopologyManagerPolicy string `json:"topologyManagerPolicy,omitempty"`
AllowedUnsafeSysctls *[]string `json:"allowedUnsafeSysctls,omitempty"`
FailSwapOn *bool `json:"failSwapOn,omitempty"`
ContainerLogMaxSizeMB *int32 `json:"containerLogMaxSizeMB,omitempty"`
ContainerLogMaxFiles *int32 `json:"containerLogMaxFiles,omitempty"`
PodMaxPids *int32 `json:"podMaxPids,omitempty"`
CPUManagerPolicy string `json:"cpuManagerPolicy,omitempty"`
CPUCfsQuota *bool `json:"cpuCfsQuota,omitempty"`
CPUCfsQuotaPeriod string `json:"cpuCfsQuotaPeriod,omitempty"`
ImageGcHighThreshold *int32 `json:"imageGcHighThreshold,omitempty"`
ImageGcLowThreshold *int32 `json:"imageGcLowThreshold,omitempty"`
TopologyManagerPolicy string `json:"topologyManagerPolicy,omitempty"`
AllowedUnsafeSysctls *[]string `json:"allowedUnsafeSysctls,omitempty"`
FailSwapOn *bool `json:"failSwapOn,omitempty"`
ContainerLogMaxSizeMB *int32 `json:"containerLogMaxSizeMB,omitempty"`
ContainerLogMaxFiles *int32 `json:"containerLogMaxFiles,omitempty"`
PodMaxPids *int32 `json:"podMaxPids,omitempty"`
ShutdownGracePeriod *int32 `json:"shutdownGracePeriod,omitempty"`
ShutdownGracePeriodCriticalPods *int32 `json:"shutdownGracePeriodCriticalPods,omitempty"`
}

// CustomLinuxOSConfig represents custom os configurations for agent pool nodes.
Expand Down Expand Up @@ -2040,6 +2042,22 @@ type AKSKubeletConfiguration struct {
Default: []
+optional. */
AllowedUnsafeSysctls []string `json:"allowedUnsafeSysctls,omitempty"`
// shutdownGracePeriod specifies the total duration that the node should delay the
// shutdown and total grace period for pod termination during a node shutdown.
// Default: "0s"
// +featureGate=GracefulNodeShutdown
// +optional
ShutdownGracePeriod Duration `json:"shutdownGracePeriod,omitempty"`
// shutdownGracePeriodCriticalPods specifies the duration used to terminate critical
// pods during a node shutdown. This should be less than shutdownGracePeriod.
// For example, if shutdownGracePeriod=30s, and shutdownGracePeriodCriticalPods=10s,
// during a node shutdown the first 20 seconds would be reserved for gracefully
// terminating normal pods, and the last 10 seconds would be reserved for terminating
// critical pods.
// Default: "0s"
// +featureGate=GracefulNodeShutdown
// +optional
ShutdownGracePeriodCriticalPods Duration `json:"shutdownGracePeriodCriticalPods,omitempty"`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Where do we validate whether ShutdownGracePeriodCriticalPods is lower than ShutdownGracePeriod?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Validation is going to be in RP, similar to ImageGcHighThreshold and ImageGcLowThreshold.

}

type Duration string
Expand Down
2 changes: 1 addition & 1 deletion pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/CSECommand
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/CSECommand
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion pkg/agent/testdata/AKSUbuntu2204+CustomKubeletConfig+CustomLinuxOSConfig/CSECommand
View file
Open in desktop

Large diffs are not rendered by default.

6 changes: 6 additions & 0 deletions pkg/agent/utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -497,6 +497,12 @@ func setCustomKubeletConfig(customKc *datamodel.CustomKubeletConfig,
if customKc.PodMaxPids != nil {
kubeletConfig.PodPidsLimit = to.Int64Ptr(int64(*customKc.PodMaxPids))
}
if customKc.ShutdownGracePeriod != nil {
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I also wanted to add some defaulting logic based on Kubernetes version. What do you think if we add IsKubernetesVersionGe(xxx) in this method? @UtheMan

kubeletConfig.ShutdownGracePeriod = datamodel.Duration(fmt.Sprintf("%ds", *customKc.ShutdownGracePeriod))
}
if customKc.ShutdownGracePeriodCriticalPods != nil {
kubeletConfig.ShutdownGracePeriodCriticalPods = datamodel.Duration(fmt.Sprintf("%ds", *customKc.ShutdownGracePeriodCriticalPods))
}
}
}

Expand Down
102 changes: 58 additions & 44 deletions pkg/agent/utils_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,17 +47,19 @@ func TestGetKubeletConfigFileFromFlags(t *testing.T) {
"--kube-reserved": "cpu=100m,memory=1638Mi",
}
customKc := &datamodel.CustomKubeletConfig{
CPUManagerPolicy: "static",
CPUCfsQuota: to.BoolPtr(false),
CPUCfsQuotaPeriod: "200ms",
ImageGcHighThreshold: to.Int32Ptr(90),
ImageGcLowThreshold: to.Int32Ptr(70),
TopologyManagerPolicy: "best-effort",
AllowedUnsafeSysctls: &[]string{"kernel.msg*", "net.ipv4.route.min_pmtu"},
FailSwapOn: to.BoolPtr(false),
ContainerLogMaxSizeMB: to.Int32Ptr(1000),
ContainerLogMaxFiles: to.Int32Ptr(99),
PodMaxPids: to.Int32Ptr(12345),
CPUManagerPolicy: "static",
CPUCfsQuota: to.BoolPtr(false),
CPUCfsQuotaPeriod: "200ms",
ImageGcHighThreshold: to.Int32Ptr(90),
ImageGcLowThreshold: to.Int32Ptr(70),
TopologyManagerPolicy: "best-effort",
AllowedUnsafeSysctls: &[]string{"kernel.msg*", "net.ipv4.route.min_pmtu"},
FailSwapOn: to.BoolPtr(false),
ContainerLogMaxSizeMB: to.Int32Ptr(1000),
ContainerLogMaxFiles: to.Int32Ptr(99),
PodMaxPids: to.Int32Ptr(12345),
ShutdownGracePeriod: to.Int32Ptr(30),
ShutdownGracePeriodCriticalPods: to.Int32Ptr(15),
}
configFileStr := GetKubeletConfigFileContent(kc, customKc)
diff := cmp.Diff(expectedKubeletJSON, configFileStr)
Expand Down Expand Up @@ -214,7 +216,9 @@ var expectedKubeletJSON = `{
"allowedUnsafeSysctls": [
"kernel.msg*",
"net.ipv4.route.min_pmtu"
]
],
"shutdownGracePeriod": "30s",
"shutdownGracePeriodCriticalPods": "15s"
}`

var expectedKubeletJSONWithNodeStatusReportFrequency = `{
Expand Down Expand Up @@ -294,7 +298,9 @@ var expectedKubeletJSONWithNodeStatusReportFrequency = `{
"allowedUnsafeSysctls": [
"kernel.msg*",
"net.ipv4.route.min_pmtu"
]
],
"shutdownGracePeriod": "30s",
"shutdownGracePeriodCriticalPods": "15s"
}`

var expectedKubeletJSONWithContainerMaxLogSizeDefaultFromFlags = `{
Expand Down Expand Up @@ -375,21 +381,25 @@ var expectedKubeletJSONWithContainerMaxLogSizeDefaultFromFlags = `{
"allowedUnsafeSysctls": [
"kernel.msg*",
"net.ipv4.route.min_pmtu"
]
],
"shutdownGracePeriod": "30s",
"shutdownGracePeriodCriticalPods": "15s"
}`

func TestGetKubeletConfigFileFlagsWithNodeStatusReportFrequency(t *testing.T) {
kc := getExampleKcWithNodeStatusReportFrequency()
customKc := &datamodel.CustomKubeletConfig{
CPUManagerPolicy: "static",
CPUCfsQuota: to.BoolPtr(false),
CPUCfsQuotaPeriod: "200ms",
ImageGcHighThreshold: to.Int32Ptr(90),
ImageGcLowThreshold: to.Int32Ptr(70),
TopologyManagerPolicy: "best-effort",
AllowedUnsafeSysctls: &[]string{"kernel.msg*", "net.ipv4.route.min_pmtu"},
FailSwapOn: to.BoolPtr(false),
PodMaxPids: to.Int32Ptr(12345),
CPUManagerPolicy: "static",
CPUCfsQuota: to.BoolPtr(false),
CPUCfsQuotaPeriod: "200ms",
ImageGcHighThreshold: to.Int32Ptr(90),
ImageGcLowThreshold: to.Int32Ptr(70),
TopologyManagerPolicy: "best-effort",
AllowedUnsafeSysctls: &[]string{"kernel.msg*", "net.ipv4.route.min_pmtu"},
FailSwapOn: to.BoolPtr(false),
PodMaxPids: to.Int32Ptr(12345),
ShutdownGracePeriod: to.Int32Ptr(30),
ShutdownGracePeriodCriticalPods: to.Int32Ptr(15),
}
configFileStr := GetKubeletConfigFileContent(kc, customKc)
diff := cmp.Diff(expectedKubeletJSONWithNodeStatusReportFrequency, configFileStr)
Expand All @@ -401,16 +411,18 @@ func TestGetKubeletConfigFileFlagsWithNodeStatusReportFrequency(t *testing.T) {
func TestGetKubeletConfigFileFromFlagsWithContainerLogMaxSize(t *testing.T) {
kc := getExampleKcWithContainerLogMaxSize()
customKc := &datamodel.CustomKubeletConfig{
CPUManagerPolicy: "static",
CPUCfsQuota: to.BoolPtr(false),
CPUCfsQuotaPeriod: "200ms",
ImageGcHighThreshold: to.Int32Ptr(90),
ImageGcLowThreshold: to.Int32Ptr(70),
TopologyManagerPolicy: "best-effort",
AllowedUnsafeSysctls: &[]string{"kernel.msg*", "net.ipv4.route.min_pmtu"},
FailSwapOn: to.BoolPtr(false),
ContainerLogMaxFiles: to.Int32Ptr(99),
PodMaxPids: to.Int32Ptr(12345),
CPUManagerPolicy: "static",
CPUCfsQuota: to.BoolPtr(false),
CPUCfsQuotaPeriod: "200ms",
ImageGcHighThreshold: to.Int32Ptr(90),
ImageGcLowThreshold: to.Int32Ptr(70),
TopologyManagerPolicy: "best-effort",
AllowedUnsafeSysctls: &[]string{"kernel.msg*", "net.ipv4.route.min_pmtu"},
FailSwapOn: to.BoolPtr(false),
ContainerLogMaxFiles: to.Int32Ptr(99),
PodMaxPids: to.Int32Ptr(12345),
ShutdownGracePeriod: to.Int32Ptr(30),
ShutdownGracePeriodCriticalPods: to.Int32Ptr(15),
}
configFileStr := GetKubeletConfigFileContent(kc, customKc)
diff := cmp.Diff(expectedKubeletJSONWithContainerMaxLogSizeDefaultFromFlags, configFileStr)
Expand All @@ -422,17 +434,19 @@ func TestGetKubeletConfigFileFromFlagsWithContainerLogMaxSize(t *testing.T) {
func TestGetKubeletConfigFileCustomKCShouldOverrideValuesPassedInKc(t *testing.T) {
kc := getExampleKcWithContainerLogMaxSize()
customKc := &datamodel.CustomKubeletConfig{
CPUManagerPolicy: "static",
CPUCfsQuota: to.BoolPtr(false),
CPUCfsQuotaPeriod: "200ms",
ImageGcHighThreshold: to.Int32Ptr(90),
ImageGcLowThreshold: to.Int32Ptr(70),
TopologyManagerPolicy: "best-effort",
AllowedUnsafeSysctls: &[]string{"kernel.msg*", "net.ipv4.route.min_pmtu"},
FailSwapOn: to.BoolPtr(false),
ContainerLogMaxFiles: to.Int32Ptr(99),
ContainerLogMaxSizeMB: to.Int32Ptr(1000),
PodMaxPids: to.Int32Ptr(12345),
CPUManagerPolicy: "static",
CPUCfsQuota: to.BoolPtr(false),
CPUCfsQuotaPeriod: "200ms",
ImageGcHighThreshold: to.Int32Ptr(90),
ImageGcLowThreshold: to.Int32Ptr(70),
TopologyManagerPolicy: "best-effort",
AllowedUnsafeSysctls: &[]string{"kernel.msg*", "net.ipv4.route.min_pmtu"},
FailSwapOn: to.BoolPtr(false),
ContainerLogMaxFiles: to.Int32Ptr(99),
ContainerLogMaxSizeMB: to.Int32Ptr(1000),
PodMaxPids: to.Int32Ptr(12345),
ShutdownGracePeriod: to.Int32Ptr(30),
ShutdownGracePeriodCriticalPods: to.Int32Ptr(15),
}
configFileStr := GetKubeletConfigFileContent(kc, customKc)
diff := cmp.Diff(expectedKubeletJSON, configFileStr)
Expand Down
Loading