fix: should not start k8s-restart-job when reboot is needed for windo…

…ws gpu (#4087)
Azure · Feb 20, 2024 · f425ed8 · f425ed8
1 parent 75004e9
commit f425ed8
Show file tree

Hide file tree

Showing 14 changed files with 266 additions and 252 deletions.
diff --git a/parts/windows/kuberneteswindowssetup.ps1 b/parts/windows/kuberneteswindowssetup.ps1
@@ -471,25 +471,26 @@ try
 
     Enable-GuestVMLogs -IntervalInMinutes $global:LogGeneratorIntervalInMinutes
 
-    Write-Log "Setup Complete, starting NodeResetScriptTask to register Winodws node without reboot"
-    Start-ScheduledTask -TaskName "k8s-restart-job"
-
-    $timeout = 180 ##  seconds
-    $timer = [Diagnostics.Stopwatch]::StartNew()
-    while ((Get-ScheduledTask -TaskName 'k8s-restart-job').State -ne 'Ready') {
-        # The task `k8s-restart-job` needs ~8 seconds.
-        if ($timer.Elapsed.TotalSeconds -gt $timeout) {
-            Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK -ErrorMessage "NodeResetScriptTask is not finished after [$($timer.Elapsed.TotalSeconds)] seconds"
-        }
-
-        Write-Log -Message "Waiting on NodeResetScriptTask..."
-        Start-Sleep -Seconds 3
-    }
-    $timer.Stop()
-    Write-Log -Message "We waited [$($timer.Elapsed.TotalSeconds)] seconds on NodeResetScriptTask"
-
-    if ($global:RebootNeeded -eq $true) {
+    if ($global:RebootNeeded) {
+        Write-Log "Setup Complete, calling Postpone-RestartComputer with reboot"
         Postpone-RestartComputer
+    } else {
+        Write-Log "Setup Complete, starting NodeResetScriptTask to register Winodws node without reboot"
+        Start-ScheduledTask -TaskName "k8s-restart-job"
+
+        $timeout = 180 ##  seconds
+        $timer = [Diagnostics.Stopwatch]::StartNew()
+        while ((Get-ScheduledTask -TaskName 'k8s-restart-job').State -ne 'Ready') {
+            # The task `k8s-restart-job` needs ~8 seconds.
+            if ($timer.Elapsed.TotalSeconds -gt $timeout) {
+                Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK -ErrorMessage "NodeResetScriptTask is not finished after [$($timer.Elapsed.TotalSeconds)] seconds"
+            }
+
+            Write-Log -Message "Waiting on NodeResetScriptTask..."
+            Start-Sleep -Seconds 3
+        }
+        $timer.Stop()
+        Write-Log -Message "We waited [$($timer.Elapsed.TotalSeconds)] seconds on NodeResetScriptTask"
     }
 }
 catch

diff --git a/pkg/agent/testdata/AKSWindows2019+CustomCloud/CustomData b/pkg/agent/testdata/AKSWindows2019+CustomCloud/CustomData
@@ -467,25 +467,26 @@ try
 
     Enable-GuestVMLogs -IntervalInMinutes $global:LogGeneratorIntervalInMinutes
 
-    Write-Log "Setup Complete, starting NodeResetScriptTask to register Winodws node without reboot"
-    Start-ScheduledTask -TaskName "k8s-restart-job"
-
-    $timeout = 180 ##  seconds
-    $timer = [Diagnostics.Stopwatch]::StartNew()
-    while ((Get-ScheduledTask -TaskName 'k8s-restart-job').State -ne 'Ready') {
-        # The task `k8s-restart-job` needs ~8 seconds.
-        if ($timer.Elapsed.TotalSeconds -gt $timeout) {
-            Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK -ErrorMessage "NodeResetScriptTask is not finished after [$($timer.Elapsed.TotalSeconds)] seconds"
-        }
-
-        Write-Log -Message "Waiting on NodeResetScriptTask..."
-        Start-Sleep -Seconds 3
-    }
-    $timer.Stop()
-    Write-Log -Message "We waited [$($timer.Elapsed.TotalSeconds)] seconds on NodeResetScriptTask"
-
-    if ($global:RebootNeeded -eq $true) {
+    if ($global:RebootNeeded) {
+        Write-Log "Setup Complete, calling Postpone-RestartComputer with reboot"
         Postpone-RestartComputer
+    } else {
+        Write-Log "Setup Complete, starting NodeResetScriptTask to register Winodws node without reboot"
+        Start-ScheduledTask -TaskName "k8s-restart-job"
+
+        $timeout = 180 ##  seconds
+        $timer = [Diagnostics.Stopwatch]::StartNew()
+        while ((Get-ScheduledTask -TaskName 'k8s-restart-job').State -ne 'Ready') {
+            # The task `k8s-restart-job` needs ~8 seconds.
+            if ($timer.Elapsed.TotalSeconds -gt $timeout) {
+                Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK -ErrorMessage "NodeResetScriptTask is not finished after [$($timer.Elapsed.TotalSeconds)] seconds"
+            }
+
+            Write-Log -Message "Waiting on NodeResetScriptTask..."
+            Start-Sleep -Seconds 3
+        }
+        $timer.Stop()
+        Write-Log -Message "We waited [$($timer.Elapsed.TotalSeconds)] seconds on NodeResetScriptTask"
     }
 }
 catch

diff --git a/pkg/agent/testdata/AKSWindows2019+CustomVnet/CustomData b/pkg/agent/testdata/AKSWindows2019+CustomVnet/CustomData
@@ -461,25 +461,26 @@ try
 
     Enable-GuestVMLogs -IntervalInMinutes $global:LogGeneratorIntervalInMinutes
 
-    Write-Log "Setup Complete, starting NodeResetScriptTask to register Winodws node without reboot"
-    Start-ScheduledTask -TaskName "k8s-restart-job"
-
-    $timeout = 180 ##  seconds
-    $timer = [Diagnostics.Stopwatch]::StartNew()
-    while ((Get-ScheduledTask -TaskName 'k8s-restart-job').State -ne 'Ready') {
-        # The task `k8s-restart-job` needs ~8 seconds.
-        if ($timer.Elapsed.TotalSeconds -gt $timeout) {
-            Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK -ErrorMessage "NodeResetScriptTask is not finished after [$($timer.Elapsed.TotalSeconds)] seconds"
-        }
-
-        Write-Log -Message "Waiting on NodeResetScriptTask..."
-        Start-Sleep -Seconds 3
-    }
-    $timer.Stop()
-    Write-Log -Message "We waited [$($timer.Elapsed.TotalSeconds)] seconds on NodeResetScriptTask"
-
-    if ($global:RebootNeeded -eq $true) {
+    if ($global:RebootNeeded) {
+        Write-Log "Setup Complete, calling Postpone-RestartComputer with reboot"
         Postpone-RestartComputer
+    } else {
+        Write-Log "Setup Complete, starting NodeResetScriptTask to register Winodws node without reboot"
+        Start-ScheduledTask -TaskName "k8s-restart-job"
+
+        $timeout = 180 ##  seconds
+        $timer = [Diagnostics.Stopwatch]::StartNew()
+        while ((Get-ScheduledTask -TaskName 'k8s-restart-job').State -ne 'Ready') {
+            # The task `k8s-restart-job` needs ~8 seconds.
+            if ($timer.Elapsed.TotalSeconds -gt $timeout) {
+                Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK -ErrorMessage "NodeResetScriptTask is not finished after [$($timer.Elapsed.TotalSeconds)] seconds"
+            }
+
+            Write-Log -Message "Waiting on NodeResetScriptTask..."
+            Start-Sleep -Seconds 3
+        }
+        $timer.Stop()
+        Write-Log -Message "We waited [$($timer.Elapsed.TotalSeconds)] seconds on NodeResetScriptTask"
     }
 }
 catch

diff --git a/pkg/agent/testdata/AKSWindows2019+EnablePrivateClusterHostsConfigAgent/CustomData b/pkg/agent/testdata/AKSWindows2019+EnablePrivateClusterHostsConfigAgent/CustomData
@@ -461,25 +461,26 @@ try
 
     Enable-GuestVMLogs -IntervalInMinutes $global:LogGeneratorIntervalInMinutes
 
-    Write-Log "Setup Complete, starting NodeResetScriptTask to register Winodws node without reboot"
-    Start-ScheduledTask -TaskName "k8s-restart-job"
-
-    $timeout = 180 ##  seconds
-    $timer = [Diagnostics.Stopwatch]::StartNew()
-    while ((Get-ScheduledTask -TaskName 'k8s-restart-job').State -ne 'Ready') {
-        # The task `k8s-restart-job` needs ~8 seconds.
-        if ($timer.Elapsed.TotalSeconds -gt $timeout) {
-            Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK -ErrorMessage "NodeResetScriptTask is not finished after [$($timer.Elapsed.TotalSeconds)] seconds"
-        }
-
-        Write-Log -Message "Waiting on NodeResetScriptTask..."
-        Start-Sleep -Seconds 3
-    }
-    $timer.Stop()
-    Write-Log -Message "We waited [$($timer.Elapsed.TotalSeconds)] seconds on NodeResetScriptTask"
-
-    if ($global:RebootNeeded -eq $true) {
+    if ($global:RebootNeeded) {
+        Write-Log "Setup Complete, calling Postpone-RestartComputer with reboot"
         Postpone-RestartComputer
+    } else {
+        Write-Log "Setup Complete, starting NodeResetScriptTask to register Winodws node without reboot"
+        Start-ScheduledTask -TaskName "k8s-restart-job"
+
+        $timeout = 180 ##  seconds
+        $timer = [Diagnostics.Stopwatch]::StartNew()
+        while ((Get-ScheduledTask -TaskName 'k8s-restart-job').State -ne 'Ready') {
+            # The task `k8s-restart-job` needs ~8 seconds.
+            if ($timer.Elapsed.TotalSeconds -gt $timeout) {
+                Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK -ErrorMessage "NodeResetScriptTask is not finished after [$($timer.Elapsed.TotalSeconds)] seconds"
+            }
+
+            Write-Log -Message "Waiting on NodeResetScriptTask..."
+            Start-Sleep -Seconds 3
+        }
+        $timer.Stop()
+        Write-Log -Message "We waited [$($timer.Elapsed.TotalSeconds)] seconds on NodeResetScriptTask"
     }
 }
 catch

diff --git a/pkg/agent/testdata/AKSWindows2019+K8S116/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S116/CustomData
@@ -461,25 +461,26 @@ try
 
     Enable-GuestVMLogs -IntervalInMinutes $global:LogGeneratorIntervalInMinutes
 
-    Write-Log "Setup Complete, starting NodeResetScriptTask to register Winodws node without reboot"
-    Start-ScheduledTask -TaskName "k8s-restart-job"
-
-    $timeout = 180 ##  seconds
-    $timer = [Diagnostics.Stopwatch]::StartNew()
-    while ((Get-ScheduledTask -TaskName 'k8s-restart-job').State -ne 'Ready') {
-        # The task `k8s-restart-job` needs ~8 seconds.
-        if ($timer.Elapsed.TotalSeconds -gt $timeout) {
-            Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK -ErrorMessage "NodeResetScriptTask is not finished after [$($timer.Elapsed.TotalSeconds)] seconds"
-        }
-
-        Write-Log -Message "Waiting on NodeResetScriptTask..."
-        Start-Sleep -Seconds 3
-    }
-    $timer.Stop()
-    Write-Log -Message "We waited [$($timer.Elapsed.TotalSeconds)] seconds on NodeResetScriptTask"
-
-    if ($global:RebootNeeded -eq $true) {
+    if ($global:RebootNeeded) {
+        Write-Log "Setup Complete, calling Postpone-RestartComputer with reboot"
         Postpone-RestartComputer
+    } else {
+        Write-Log "Setup Complete, starting NodeResetScriptTask to register Winodws node without reboot"
+        Start-ScheduledTask -TaskName "k8s-restart-job"
+
+        $timeout = 180 ##  seconds
+        $timer = [Diagnostics.Stopwatch]::StartNew()
+        while ((Get-ScheduledTask -TaskName 'k8s-restart-job').State -ne 'Ready') {
+            # The task `k8s-restart-job` needs ~8 seconds.
+            if ($timer.Elapsed.TotalSeconds -gt $timeout) {
+                Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK -ErrorMessage "NodeResetScriptTask is not finished after [$($timer.Elapsed.TotalSeconds)] seconds"
+            }
+
+            Write-Log -Message "Waiting on NodeResetScriptTask..."
+            Start-Sleep -Seconds 3
+        }
+        $timer.Stop()
+        Write-Log -Message "We waited [$($timer.Elapsed.TotalSeconds)] seconds on NodeResetScriptTask"
     }
 }
 catch

diff --git a/pkg/agent/testdata/AKSWindows2019+K8S117/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S117/CustomData
@@ -461,25 +461,26 @@ try
 
     Enable-GuestVMLogs -IntervalInMinutes $global:LogGeneratorIntervalInMinutes
 
-    Write-Log "Setup Complete, starting NodeResetScriptTask to register Winodws node without reboot"
-    Start-ScheduledTask -TaskName "k8s-restart-job"
-
-    $timeout = 180 ##  seconds
-    $timer = [Diagnostics.Stopwatch]::StartNew()
-    while ((Get-ScheduledTask -TaskName 'k8s-restart-job').State -ne 'Ready') {
-        # The task `k8s-restart-job` needs ~8 seconds.
-        if ($timer.Elapsed.TotalSeconds -gt $timeout) {
-            Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK -ErrorMessage "NodeResetScriptTask is not finished after [$($timer.Elapsed.TotalSeconds)] seconds"
-        }
-
-        Write-Log -Message "Waiting on NodeResetScriptTask..."
-        Start-Sleep -Seconds 3
-    }
-    $timer.Stop()
-    Write-Log -Message "We waited [$($timer.Elapsed.TotalSeconds)] seconds on NodeResetScriptTask"
-
-    if ($global:RebootNeeded -eq $true) {
+    if ($global:RebootNeeded) {
+        Write-Log "Setup Complete, calling Postpone-RestartComputer with reboot"
         Postpone-RestartComputer
+    } else {
+        Write-Log "Setup Complete, starting NodeResetScriptTask to register Winodws node without reboot"
+        Start-ScheduledTask -TaskName "k8s-restart-job"
+
+        $timeout = 180 ##  seconds
+        $timer = [Diagnostics.Stopwatch]::StartNew()
+        while ((Get-ScheduledTask -TaskName 'k8s-restart-job').State -ne 'Ready') {
+            # The task `k8s-restart-job` needs ~8 seconds.
+            if ($timer.Elapsed.TotalSeconds -gt $timeout) {
+                Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK -ErrorMessage "NodeResetScriptTask is not finished after [$($timer.Elapsed.TotalSeconds)] seconds"
+            }
+
+            Write-Log -Message "Waiting on NodeResetScriptTask..."
+            Start-Sleep -Seconds 3
+        }
+        $timer.Stop()
+        Write-Log -Message "We waited [$($timer.Elapsed.TotalSeconds)] seconds on NodeResetScriptTask"
     }
 }
 catch

diff --git a/pkg/agent/testdata/AKSWindows2019+K8S118/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S118/CustomData
@@ -461,25 +461,26 @@ try
 
     Enable-GuestVMLogs -IntervalInMinutes $global:LogGeneratorIntervalInMinutes
 
-    Write-Log "Setup Complete, starting NodeResetScriptTask to register Winodws node without reboot"
-    Start-ScheduledTask -TaskName "k8s-restart-job"
-
-    $timeout = 180 ##  seconds
-    $timer = [Diagnostics.Stopwatch]::StartNew()
-    while ((Get-ScheduledTask -TaskName 'k8s-restart-job').State -ne 'Ready') {
-        # The task `k8s-restart-job` needs ~8 seconds.
-        if ($timer.Elapsed.TotalSeconds -gt $timeout) {
-            Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK -ErrorMessage "NodeResetScriptTask is not finished after [$($timer.Elapsed.TotalSeconds)] seconds"
-        }
-
-        Write-Log -Message "Waiting on NodeResetScriptTask..."
-        Start-Sleep -Seconds 3
-    }
-    $timer.Stop()
-    Write-Log -Message "We waited [$($timer.Elapsed.TotalSeconds)] seconds on NodeResetScriptTask"
-
-    if ($global:RebootNeeded -eq $true) {
+    if ($global:RebootNeeded) {
+        Write-Log "Setup Complete, calling Postpone-RestartComputer with reboot"
         Postpone-RestartComputer
+    } else {
+        Write-Log "Setup Complete, starting NodeResetScriptTask to register Winodws node without reboot"
+        Start-ScheduledTask -TaskName "k8s-restart-job"
+
+        $timeout = 180 ##  seconds
+        $timer = [Diagnostics.Stopwatch]::StartNew()
+        while ((Get-ScheduledTask -TaskName 'k8s-restart-job').State -ne 'Ready') {
+            # The task `k8s-restart-job` needs ~8 seconds.
+            if ($timer.Elapsed.TotalSeconds -gt $timeout) {
+                Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK -ErrorMessage "NodeResetScriptTask is not finished after [$($timer.Elapsed.TotalSeconds)] seconds"
+            }
+
+            Write-Log -Message "Waiting on NodeResetScriptTask..."
+            Start-Sleep -Seconds 3
+        }
+        $timer.Stop()
+        Write-Log -Message "We waited [$($timer.Elapsed.TotalSeconds)] seconds on NodeResetScriptTask"
     }
 }
 catch

diff --git a/pkg/agent/testdata/AKSWindows2019+K8S119+CSI/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S119+CSI/CustomData
@@ -461,25 +461,26 @@ try
 
     Enable-GuestVMLogs -IntervalInMinutes $global:LogGeneratorIntervalInMinutes
 
-    Write-Log "Setup Complete, starting NodeResetScriptTask to register Winodws node without reboot"
-    Start-ScheduledTask -TaskName "k8s-restart-job"
-
-    $timeout = 180 ##  seconds
-    $timer = [Diagnostics.Stopwatch]::StartNew()
-    while ((Get-ScheduledTask -TaskName 'k8s-restart-job').State -ne 'Ready') {
-        # The task `k8s-restart-job` needs ~8 seconds.
-        if ($timer.Elapsed.TotalSeconds -gt $timeout) {
-            Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK -ErrorMessage "NodeResetScriptTask is not finished after [$($timer.Elapsed.TotalSeconds)] seconds"
-        }
-
-        Write-Log -Message "Waiting on NodeResetScriptTask..."
-        Start-Sleep -Seconds 3
-    }
-    $timer.Stop()
-    Write-Log -Message "We waited [$($timer.Elapsed.TotalSeconds)] seconds on NodeResetScriptTask"
-
-    if ($global:RebootNeeded -eq $true) {
+    if ($global:RebootNeeded) {
+        Write-Log "Setup Complete, calling Postpone-RestartComputer with reboot"
         Postpone-RestartComputer
+    } else {
+        Write-Log "Setup Complete, starting NodeResetScriptTask to register Winodws node without reboot"
+        Start-ScheduledTask -TaskName "k8s-restart-job"
+
+        $timeout = 180 ##  seconds
+        $timer = [Diagnostics.Stopwatch]::StartNew()
+        while ((Get-ScheduledTask -TaskName 'k8s-restart-job').State -ne 'Ready') {
+            # The task `k8s-restart-job` needs ~8 seconds.
+            if ($timer.Elapsed.TotalSeconds -gt $timeout) {
+                Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK -ErrorMessage "NodeResetScriptTask is not finished after [$($timer.Elapsed.TotalSeconds)] seconds"
+            }
+
+            Write-Log -Message "Waiting on NodeResetScriptTask..."
+            Start-Sleep -Seconds 3
+        }
+        $timer.Stop()
+        Write-Log -Message "We waited [$($timer.Elapsed.TotalSeconds)] seconds on NodeResetScriptTask"
     }
 }
 catch