Add routing label to sandbox pods

pkg/builder/builder.go

@@ -1511,7 +1511,7 @@ func BuildStsSpec(nm types.NamespacedName, vdb *vapi.VerticaDB, sc *vapi.Subclus
 }
 
 // BuildSandboxConfigMap builds a config map for sandbox controller
-func BuildSandboxConfigMap(nm types.NamespacedName, vdb *vapi.VerticaDB, sandbox string) *corev1.ConfigMap {
+func BuildSandboxConfigMap(nm types.NamespacedName, vdb *vapi.VerticaDB, sandbox string, forUpgrade bool) *corev1.ConfigMap {
 	immutable := true
 	return &corev1.ConfigMap{
 		TypeMeta: metav1.TypeMeta{
@@ -1522,7 +1522,7 @@ func BuildSandboxConfigMap(nm types.NamespacedName, vdb *vapi.VerticaDB, sandbox
 			Name:            nm.Name,
 			Namespace:       nm.Namespace,
 			Labels:          MakeLabelsForSandboxConfigMap(vdb),
-			Annotations:     MakeAnnotationsForSandboxConfigMap(vdb),
+			Annotations:     MakeAnnotationsForSandboxConfigMap(vdb, forUpgrade),
 			OwnerReferences: []metav1.OwnerReference{vdb.GenerateOwnerReference()},
 		},
 		// the data should be immutable since dbName and sandboxName are fixed

pkg/builder/labels_annotations.go

@@ -179,8 +179,11 @@ func MakeAnnotationsForVProxyObject(vdb *vapi.VerticaDB) map[string]string {
 
 // MakeAnnotationsForSandboxConfigMap builds the list of annotations that are included
 // in the sandbox config map.
-func MakeAnnotationsForSandboxConfigMap(vdb *vapi.VerticaDB) map[string]string {
+func MakeAnnotationsForSandboxConfigMap(vdb *vapi.VerticaDB, forUpgrade bool) map[string]string {
 	annotations := MakeAnnotationsForObject(vdb)
+	if forUpgrade {
+		annotations[vmeta.DisableRoutingAnnotation] = "true"
+	}
 	if ver, ok := vdb.Annotations[vmeta.VersionAnnotation]; ok {
 		annotations[vmeta.VersionAnnotation] = ver
 	}

pkg/controllers/sandbox/sandbox_controller.go

@@ -174,6 +174,9 @@ func (r *SandboxConfigMapReconciler) constructActors(vdb *v1.VerticaDB, log logr
 		// Update the vdb status including subclusters[].shutdown, after a stop_db, stop_sc
 		// or a restart
 		vdbcontroller.MakeStatusReconcilerWithShutdown(r.Client, r.Scheme, log, vdb, pfacts),
+		// Ensure we add labels to any pod rescheduled so that Service objects route traffic to it.
+		vdbcontroller.MakeClientRoutingLabelReconcilerWithDisableRouting(r, log, vdb, pfacts, vdbcontroller.PodRescheduleApplyMethod, "",
+			vmeta.GetDisableRouting(configMap.Annotations)),
 		// Scale down the subclusters' statefulsets to zero after the subclusters are shut down
 		MakeScaleStafulsetReconciler(r, vdb, pfacts),
 	}

pkg/controllers/sandbox/unsandboxsubcluster_reconciler_test.go

@@ -33,7 +33,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 )
 
-var _ = Describe("sandboxsubcluster_reconcile", func() {
+var _ = Describe("unsandboxsubcluster_reconcile", func() {
 	ctx := context.Background()
 	maincluster := "main"
 	subcluster1 := "sc1"
@@ -92,7 +92,7 @@ var _ = Describe("sandboxsubcluster_reconcile", func() {
 		test.CreateVDB(ctx, k8sClient, vdb)
 		defer test.DeleteVDB(ctx, k8sClient, vdb)
 		nm := names.GenSandboxConfigMapName(vdb, sandbox1)
-		cm := builder.BuildSandboxConfigMap(nm, vdb, sandbox1)
+		cm := builder.BuildSandboxConfigMap(nm, vdb, sandbox1, false)
 		Expect(k8sClient.Create(ctx, cm)).Should(Succeed())
 		defer test.DeleteConfigMap(ctx, k8sClient, vdb, sandbox1)
 
@@ -127,7 +127,7 @@ var _ = Describe("sandboxsubcluster_reconcile", func() {
 		test.CreatePods(ctx, k8sClient, vdb, test.AllPodsRunning)
 		defer test.DeletePods(ctx, k8sClient, vdb)
 		nm := names.GenSandboxConfigMapName(vdb, sandbox1)
-		cm := builder.BuildSandboxConfigMap(nm, vdb, sandbox1)
+		cm := builder.BuildSandboxConfigMap(nm, vdb, sandbox1, false)
 		Expect(k8sClient.Create(ctx, cm)).Should(Succeed())
 		test.CreateVDB(ctx, k8sClient, vdb)
 		defer test.DeleteVDB(ctx, k8sClient, vdb)
@@ -201,7 +201,7 @@ var _ = Describe("sandboxsubcluster_reconcile", func() {
 		test.CreatePods(ctx, k8sClient, vdb, test.AllPodsRunning)
 		defer test.DeletePods(ctx, k8sClient, vdb)
 		nm := names.GenSandboxConfigMapName(vdb, sandbox1)
-		cm := builder.BuildSandboxConfigMap(nm, vdb, sandbox1)
+		cm := builder.BuildSandboxConfigMap(nm, vdb, sandbox1, false)
 		Expect(k8sClient.Create(ctx, cm)).Should(Succeed())
 		test.CreateVDB(ctx, k8sClient, vdb)
 		defer test.DeleteVDB(ctx, k8sClient, vdb)

pkg/controllers/vdb/clientroutinglabel_reconciler.go

@@ -51,12 +51,13 @@ const (
 )
 
 type ClientRoutingLabelReconciler struct {
-	Rec         config.ReconcilerInterface
-	Vdb         *vapi.VerticaDB // Vdb is the CRD we are acting on.
-	Log         logr.Logger
-	PFacts      *podfacts.PodFacts
-	ApplyMethod ApplyMethodType
-	ScName      string // Subcluster we are going to reconcile.  Blank if all subclusters.
+	Rec            config.ReconcilerInterface
+	Vdb            *vapi.VerticaDB // Vdb is the CRD we are acting on.
+	Log            logr.Logger
+	PFacts         *podfacts.PodFacts
+	ApplyMethod    ApplyMethodType
+	ScName         string // Subcluster we are going to reconcile.  Blank if all subclusters.
+	DisableRouting bool
 }
 
 func MakeClientRoutingLabelReconciler(recon config.ReconcilerInterface, log logr.Logger,
@@ -71,6 +72,15 @@ func MakeClientRoutingLabelReconciler(recon config.ReconcilerInterface, log logr
 	}
 }
 
+func MakeClientRoutingLabelReconcilerWithDisableRouting(recon config.ReconcilerInterface, log logr.Logger,
+	vdb *vapi.VerticaDB, pfacts *podfacts.PodFacts, applyMethod ApplyMethodType, scName string,
+	disableRouting bool) controllers.ReconcileActor {
+	act := MakeClientRoutingLabelReconciler(recon, log, vdb, pfacts, applyMethod, scName)
+	c := act.(*ClientRoutingLabelReconciler)
+	c.DisableRouting = disableRouting
+	return c
+}
+
 // Reconcile will add or remove labels that control whether it accepts client
 // connections.  Pods that have at least one shard owned will have a label added
 // so that it receives traffic.  For pods that don't own a shard or about to be
@@ -240,7 +250,7 @@ func (c *ClientRoutingLabelReconciler) manipulateRoutingLabelInPod(pod *corev1.P
 	// entire subcluster, so pending delete isn't checked.
 	switch c.ApplyMethod {
 	case AddNodeApplyMethod, PodRescheduleApplyMethod:
-		if !labelExists && pf.GetUpNode() && (pf.GetShardSubscriptions() > 0 || !c.Vdb.IsEON()) && !pf.GetIsPendingDelete() {
+		if !c.DisableRouting && !labelExists && pf.GetUpNode() && (pf.GetShardSubscriptions() > 0 || !c.Vdb.IsEON()) && !pf.GetIsPendingDelete() {
 			pod.Labels[vmeta.ClientRoutingLabel] = vmeta.ClientRoutingVal
 			c.Log.Info("Adding client routing label", "pod",
 				pod.Name, "label", fmt.Sprintf("%s=%s", vmeta.ClientRoutingLabel, vmeta.ClientRoutingVal))

pkg/controllers/vdb/clientroutinglabel_reconciler_test.go

@@ -161,4 +161,28 @@ var _ = Describe("clientroutinglabel_reconcile", func() {
 		_, ok = pod.Labels[vmeta.ClientRoutingLabel]
 		Expect(ok).Should(BeFalse())
 	})
+
+	It("should not set the annotation when disableRouting is true", func() {
+		vdb := vapi.MakeVDB()
+		vdb.Spec.Subclusters = []vapi.Subcluster{
+			{Name: "sc1", Size: 1},
+		}
+		test.CreatePods(ctx, k8sClient, vdb, test.AllPodsRunning)
+		defer test.DeletePods(ctx, k8sClient, vdb)
+
+		fpr := &cmds.FakePodRunner{}
+		pfacts := podfacts.MakePodFacts(vdbRec, fpr, logger, TestPassword)
+		Expect(pfacts.Collect(ctx, vdb)).Should(Succeed())
+		pn := names.GenPodName(vdb, &vdb.Spec.Subclusters[0], 0)
+		pfacts.Detail[pn].SetUpNode(true)
+		pfacts.Detail[pn].SetShardSubscriptions(10)
+		act := MakeClientRoutingLabelReconcilerWithDisableRouting(vdbRec, logger, vdb, &pfacts, AddNodeApplyMethod, "", true)
+		r := act.(*ClientRoutingLabelReconciler)
+		Expect(r.Reconcile(ctx, &ctrl.Request{})).Should(Equal(ctrl.Result{}))
+
+		pod := &corev1.Pod{}
+		Expect(k8sClient.Get(ctx, pn, pod)).Should(Succeed())
+		_, ok := pod.Labels[vmeta.ClientRoutingLabel]
+		Expect(ok).Should(BeFalse())
+	})
 })

pkg/controllers/vdb/onlineupgrade_reconciler.go

@@ -1052,6 +1052,13 @@ func (r *OnlineUpgradeReconciler) redirectConnectionsToReplicaGroupB(ctx context
 	if verrors.IsReconcileAborted(res, err) {
 		return res, err
 	}
+	// Now that routing to the sandbox is allowed, we no longer need
+	// disableRouting annotation so we can safely turn it off.
+	sbMan := MakeSandboxConfigMapManager(r.VRec, r.VDB, r.sandboxName, "" /*no uuid*/)
+	_, err = sbMan.turnOffDisableRoutingAnnotation(ctx)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
 	// then remove client routing labels from replica group a so no traffic is routed to the old main cluster
 	methodType := DrainNodeApplyMethod
 	if vmeta.UseVProxy(r.VDB.Annotations) {

pkg/controllers/vdb/sandbox_configmap.go

@@ -85,6 +85,20 @@ func (s *SandboxConfigMapManager) triggerSandboxController(ctx context.Context,
 	return vk8s.MetaUpdate(ctx, s.vrec.GetClient(), nm, s.configMap, chgs)
 }
 
+// turnOffDisableRoutingAnnotation sets the disable routing annotation in the configMap to false.
+func (s *SandboxConfigMapManager) turnOffDisableRoutingAnnotation(ctx context.Context) (bool, error) {
+	if err := s.fetchConfigMap(ctx); err != nil {
+		return false, err
+	}
+	anns := make(map[string]string)
+	anns[vmeta.DisableRoutingAnnotation] = "false"
+	chgs := vk8s.MetaChanges{
+		NewAnnotations: anns,
+	}
+	nm := names.GenSandboxConfigMapName(s.vdb, s.sandbox)
+	return vk8s.MetaUpdate(ctx, s.vrec.GetClient(), nm, s.configMap, chgs)
+}
+
 // fetchConfigMap will fetch the sandbox configmap
 func (s *SandboxConfigMapManager) fetchConfigMap(ctx context.Context) error {
 	nm := names.GenSandboxConfigMapName(s.vdb, s.sandbox)

pkg/controllers/vdb/sandboxsubcluster_reconciler.go

@@ -289,7 +289,7 @@ func (s *SandboxSubclusterReconciler) findInitiatorIPs(ctx context.Context, sand
 func (s *SandboxSubclusterReconciler) checkSandboxConfigMap(ctx context.Context, sandbox string) error {
 	nm := names.GenSandboxConfigMapName(s.Vdb, sandbox)
 	curCM := &corev1.ConfigMap{}
-	newCM := builder.BuildSandboxConfigMap(nm, s.Vdb, sandbox)
+	newCM := builder.BuildSandboxConfigMap(nm, s.Vdb, sandbox, s.ForUpgrade)
 	err := s.Client.Get(ctx, nm, curCM)
 	if err != nil && kerrors.IsNotFound(err) {
 		s.Log.Info("Creating sandbox config map", "Name", nm)

pkg/controllers/vdb/unsandboxsubcluster_reconciler_test.go

@@ -78,7 +78,7 @@ var _ = Describe("unsandboxsubcluster_reconcile", func() {
 		test.CreatePods(ctx, k8sClient, vdb, test.AllPodsRunning)
 		defer test.DeletePods(ctx, k8sClient, vdb)
 		nm := names.GenSandboxConfigMapName(vdb, sandbox1)
-		cm := builder.BuildSandboxConfigMap(nm, vdb, sandbox1)
+		cm := builder.BuildSandboxConfigMap(nm, vdb, sandbox1, false)
 		Expect(k8sClient.Create(ctx, cm)).Should(Succeed())
 		defer test.DeleteConfigMap(ctx, k8sClient, vdb, sandbox1)
 		test.CreateVDB(ctx, k8sClient, vdb)

pkg/meta/annotations.go

@@ -373,6 +373,10 @@ const (
 	ReplicationDefaultTimeout             = 60 * 60
 	ReplicationPollingFrequencyAnnotation = "vertica.com/replication-polling-frequency"
 	ReplicationDefaultPollingFrequency    = 5
+
+	// Annotation set in a sandbox configMap. Indicates that routing must be disabled
+	// on the sandbox nodes.
+	DisableRoutingAnnotation = "vertica.com/disable-routing"
 )
 
 // IsPauseAnnotationSet will check the annotations for a special value that will
@@ -720,6 +724,12 @@ func GetReplicationPollingFrequency(annotations map[string]string) int {
 	return lookupIntAnnotation(annotations, ReplicationPollingFrequencyAnnotation, ReplicationDefaultPollingFrequency)
 }
 
+// GetDisableRouting returns true if routing must be disabled on the sandbox
+// nodes.
+func GetDisableRouting(annotations map[string]string) bool {
+	return lookupBoolAnnotation(annotations, DisableRoutingAnnotation, false)
+}
+
 // lookupBoolAnnotation is a helper function to lookup a specific annotation and
 // treat it as if it were a boolean.
 func lookupBoolAnnotation(annotations map[string]string, annotation string, defaultValue bool) bool {

tests/e2e-leg-10/sandbox-restart/57-check-connection.yaml

@@ -0,0 +1,19 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |
+      SERVICE_DNS=v-sandbox-restart-sec1.$NAMESPACE.svc.cluster.local
+      kubectl exec -i v-sandbox-restart-pri1-0 -n $NAMESPACE -c server -- bash -c "vsql -h $SERVICE_DNS -c 'select 1;'" 2> /dev/null || exit 1