Send script kill to busy nodes

Author: secwall

internal/app/repair.go

@@ -143,6 +143,12 @@ func (app *App) repairLocalNode(master string) bool {
 			app.nodeFailTime[local.FQDN()] = time.Now()
 		}
 		failedTime := time.Since(app.nodeFailTime[local.FQDN()])
+		if failedTime > app.config.Valkey.BusyTimeout && strings.HasPrefix(err.Error(), "BUSY ") {
+			err = local.ScriptKill(app.ctx)
+			if err != nil {
+				app.logger.Error("Local node is busy running a script. But SCRIPT KILL failed", "error", err)
+			}
+		}
 		if failedTime > app.config.Valkey.RestartTimeout && !strings.HasPrefix(err.Error(), "LOADING ") {
 			app.nodeFailTime[local.FQDN()] = time.Now()
 			err = local.Restart(app.ctx)

internal/config/config.go

@@ -25,6 +25,7 @@ type ValkeyConfig struct {
 	WriteTimeout            time.Duration `yaml:"write_timeout"`
 	DNSTTL                  time.Duration `yaml:"dns_ttl"`
 	FailoverTimeout         time.Duration `yaml:"failover_timeout"`
+	BusyTimeout             time.Duration `yaml:"busy_timeout"`
 	Port                    int           `yaml:"port"`
 	RestartTimeout          time.Duration `yaml:"restart_timeout"`
 	WaitReplicationTimeout  time.Duration `yaml:"wait_replication_timeout"`
@@ -99,6 +100,7 @@ func DefaultValkeyConfig() ValkeyConfig {
 		WaitPoisonPillTimeout:   30 * time.Second,
 		StaleReplicaLagClose:    60 * time.Second,
 		StaleReplicaLagOpen:     10 * time.Second,
+		BusyTimeout:             5 * time.Second,
 		MaxParallelSyncs:        1,
 		AllowDataLoss:           false,
 		TurnBeforeSwitchover:    false,

internal/valkey/node.go

@@ -703,3 +703,12 @@ func (n *Node) HasClusterSlots(ctx context.Context) (bool, error) {
 	}
 	return false, nil
 }
+
+// ScriptKill kills a running script if node is in BUSY state
+func (n *Node) ScriptKill(ctx context.Context) error {
+	err := n.ensureConn()
+	if err != nil {
+		return err
+	}
+	return n.conn.Do(ctx, n.conn.B().ScriptKill().Build()).Error()
+}

tests/features/07_cluster_local_repair.feature

@@ -1,5 +1,22 @@
 Feature: Cluster mode local node repair
 
+    Scenario: Busy cluster node gets a SCRIPT KILL
+        Given clustered shard is up and running
+        Then valkey host "valkey1" should be master
+        And valkey host "valkey2" should become replica of "valkey1" within "15" seconds
+        And replication on valkey host "valkey2" should run fine within "15" seconds
+        And valkey host "valkey3" should become replica of "valkey1" within "15" seconds
+        And replication on valkey host "valkey3" should run fine within "15" seconds
+        And zookeeper node "/test/active_nodes" should match json_exactly within "30" seconds
+        """
+            ["valkey1","valkey2","valkey3"]
+        """
+        When I run async command on host "valkey1"
+        """
+            valkey-cli -a functestpassword eval 'while true do end' 0
+        """
+        Then valkey host "valkey1" should become available within "60" seconds
+
     Scenario: Cluster mode replica is restarted after OOM
         Given clustered shard is up and running
         Then valkey host "valkey1" should be master

tests/features/07_sentinel_local_repair.feature

@@ -17,6 +17,23 @@ Feature: Sentinel mode local node repair
         """
         Then senticache host "valkey1" should have master "valkey1" within "30" seconds
 
+    Scenario: Busy sentinel mode node gets a SCRIPT KILL
+        Given sentinel shard is up and running
+        Then valkey host "valkey1" should be master
+        And valkey host "valkey2" should become replica of "valkey1" within "15" seconds
+        And replication on valkey host "valkey2" should run fine within "15" seconds
+        And valkey host "valkey3" should become replica of "valkey1" within "15" seconds
+        And replication on valkey host "valkey3" should run fine within "15" seconds
+        And zookeeper node "/test/active_nodes" should match json_exactly within "30" seconds
+        """
+            ["valkey1","valkey2","valkey3"]
+        """
+        When I run async command on host "valkey1"
+        """
+            valkey-cli -a functestpassword eval 'while true do end' 0
+        """
+        Then valkey host "valkey1" should become available within "60" seconds
+
     Scenario: Sentinel mode replica is restarted after OOM
         Given sentinel shard is up and running
         Then valkey host "valkey1" should be master