Allows Restores Across Namespaces (CrunchyData#2358)

When creating a new PostgreSQL cluster via 'pgbackrest restore' using
the pgBackRest backups from another current or former PostgreSQL
cluster, it is now possible restore from the backups of a cluster in
different namespace (i.e. different than the cluster being created).

In support of this functionality, a copy of the pgBackRest Secret
corresponding to the repository containing the backups being utilized
to bootstrap the cluster is now made specifically for the bootstrap Job
(and then removed when bootstrapping is complete).  This ensures the
new cluster always has the information required to connect to the
repository being utilized to bootstrap the cluster, regardless of what
namespace it is in.  Additionally, DNS for the pgBackRest repository
service is now always used when performing the restore to ensure
proper connectivity across namespaces if/when needed.

And finally, a  '--restore-from-namespace' option has been added to the
pgo client, which corresponds to a 'spec.pgdatasource.namespace' field
in the Pgcluster spec, and allows users to specify the namespace of the
cluster specified for bootstrapping the new cluster.

Issue: [ch9969]
Issue: CrunchyData#2062

Author: andrewlecuyer

cmd/pgo/cmd/cluster.go

@@ -332,6 +332,7 @@ func createCluster(args []string, ns string, createClusterCmd *cobra.Command) {
 	r.WALStorageConfig = WALStorageConfig
 	r.WALPVCSize = WALPVCSize
 	r.PGDataSource.RestoreFrom = RestoreFrom
+	r.PGDataSource.Namespace = RestoreFromNamespace
 	r.PGDataSource.RestoreOpts = BackupOpts
 	// set any annotations
 	r.Annotations = getClusterAnnotations(Annotations, AnnotationsPostgres, AnnotationsBackrest,

cmd/pgo/cmd/create.go

@@ -71,6 +71,7 @@ var (
 	WALStorageConfig                                                                             string
 	WALPVCSize                                                                                   string
 	RestoreFrom                                                                                  string
+	RestoreFromNamespace                                                                         string
 )
 
 // group the annotation requests
@@ -486,6 +487,9 @@ func init() {
 		"the TLS keypair to use for enabling certificate-based authentication between PostgreSQL instances, "+
 		"particularly for the purpose of replication. Must be used with \"server-tls-secret\" and \"server-ca-secret\".")
 	createClusterCmd.Flags().StringVarP(&RestoreFrom, "restore-from", "", "", "The name of cluster to restore from when bootstrapping a new cluster")
+	createClusterCmd.Flags().StringVarP(&RestoreFromNamespace, "restore-from-namespace", "", "",
+		"The namespace for the cluster specified using --restore-from.  Defaults to the "+
+			"namespace of the cluster being created if not provided.")
 	createClusterCmd.Flags().StringVarP(&BackupOpts, "restore-opts", "", "",
 		"The options to pass into pgbackrest where performing a restore to bootrap the cluster. "+
 			"Only applicable when a \"restore-from\" value is specified")

docs/content/architecture/disaster-recovery.md

@@ -121,6 +121,9 @@ command with several flags:
 - `--restore-from`: specifies the name of a PostgreSQL cluster (either one that
 is active, or a former cluster whose pgBackRest repository still exists) to
 restore from.
+- `--restore-from-namespace` (optional): the namespace of the PostgreSQL cluster specified
+using `--restore-from` (the namespace of the cluster being created is utilized if a namespace
+is not specified using this option)
 - `--restore-opts`: used to specify additional options, similar to the ones that
 are passed into [`pgbackrest restore`](https://pgbackrest.org/command.html#command-restore).
 
@@ -143,7 +146,10 @@ pgo create cluster newcluster \
 
 Note that when using this method, the PostgreSQL Operator can only restore one
 cluster from each pgBackRest repository at a time. Using the above example, one
-can only perform one restore from `oldcluster` at a given time.
+can only perform one restore from `oldcluster` at a given time.  Additionally,
+if the cluster being utilized for restore is in another namespace than the
+cluster being created, the proper namespace can be specified using the
+`--restore-from-namespace` option.
 
 When using the restore to a new cluster method, the PostgreSQL Operator takes
 the following actions:

docs/content/architecture/provisioning.md

@@ -173,7 +173,9 @@ the existing pgBackRest repository host for that cluster in order to perform the
 restore.  If restoring from a former cluster that has since been deleted, a new pgBackRest
 repository host will be deployed for the sole purpose of bootstrapping the new cluster, and will
 then be destroyed once the restore is complete.  Also, please note that it is only possible for
-one cluster to bootstrap from another cluster (whether running or not) at any given time.
+one cluster to bootstrap from another cluster (whether running or not) at any given time.  And
+finally, if the cluster being utilized for restore is in another namespace than the cluster being
+created, the proper namespace can be specified using the `--restore-from-namespace` option.
 
 ## Deprovisioning
 

docs/content/pgo-client/common-tasks.md

@@ -645,6 +645,15 @@ execute the following command:
 pgo create cluster newcluster --restore-from=oldcluster
 ```
 
+##### Full Restore Across Namespaces
+
+To create a new PostgreSQL cluster from a backup in another namespace and restore it
+fully, you can execute the following command:
+
+```
+pgo create cluster newcluster --restore-from=oldcluster --restore-from-namespace=oldnamespace
+```
+
 ##### Point-in-time-Recovery (PITR)
 
 To create a new PostgreSQL cluster and restore it to specific point-in-time

installers/ansible/roles/pgo-operator/files/pgo-configs/cluster-bootstrap-job.json

@@ -185,7 +185,7 @@
                 }, {
                     "name": "sshd",
                     "secret": {
-                        "secretName": "{{.RestoreFrom}}-backrest-repo-config"
+                        "secretName": "{{.ClusterName}}-bootstrap-backrest-repo-config"
                     }
                 }, {
                     "name": "ssh-config",

installers/ansible/roles/pgo-operator/files/pgo-configs/pgo-backrest-repo-template.json

@@ -6,6 +6,7 @@
         "labels": {
             {{if .BootstrapCluster}}
             "pgha-bootstrap": "{{.BootstrapCluster}}",
+            "pgha-bootstrap-namespace": "{{.BootstrapNamespace}}",
             {{ end }}
             "name": "{{.Name}}",
             "pg-cluster": "{{.ClusterName}}",
@@ -34,6 +35,7 @@
                 "labels": {
                     {{if .BootstrapCluster}}
                     "pgha-bootstrap": "{{.BootstrapCluster}}",
+                    "pgha-bootstrap-namespace": "{{.BootstrapNamespace}}",
                     {{ end }}
                     "name": "{{.Name}}",
                     "pg-cluster": "{{.ClusterName}}",

internal/apiserver/clusterservice/clusterimpl.go

@@ -2335,10 +2335,17 @@ func isMissingExistingDataSourceS3Config(backrestRepoSecret *v1.Secret) bool {
 // to create a new cluster
 func validateDataSourceParms(request *msgs.CreateClusterRequest) error {
 	ctx := context.TODO()
-	namespace := request.Namespace
+
 	restoreClusterName := request.PGDataSource.RestoreFrom
 	restoreOpts := request.PGDataSource.RestoreOpts
 
+	var restoreFromNamespace string
+	if request.PGDataSource.Namespace != "" {
+		restoreFromNamespace = request.PGDataSource.Namespace
+	} else {
+		restoreFromNamespace = request.Namespace
+	}
+
 	if restoreClusterName == "" && restoreOpts == "" {
 		return nil
 	}
@@ -2351,21 +2358,23 @@ func validateDataSourceParms(request *msgs.CreateClusterRequest) error {
 	}
 
 	// next verify whether or not a PVC exists for the cluster we are restoring from
-	if _, err := apiserver.Clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx,
+	if _, err := apiserver.Clientset.CoreV1().PersistentVolumeClaims(restoreFromNamespace).Get(ctx,
 		fmt.Sprintf(util.BackrestRepoPVCName, restoreClusterName),
 		metav1.GetOptions{}); err != nil {
-		return fmt.Errorf("Unable to find PVC %s for cluster %s, cannot to restore from the "+
-			"specified data source", fmt.Sprintf(util.BackrestRepoPVCName, restoreClusterName),
-			restoreClusterName)
+		return fmt.Errorf("Unable to find PVC %s for cluster %s (namespace %s), cannot restore "+
+			"from the specified data source",
+			fmt.Sprintf(util.BackrestRepoPVCName, restoreClusterName),
+			restoreClusterName, restoreFromNamespace)
 	}
 
 	// now verify that a pgBackRest repo secret exists for the cluster we are restoring from
-	backrestRepoSecret, err := apiserver.Clientset.CoreV1().Secrets(namespace).Get(ctx,
+	backrestRepoSecret, err := apiserver.Clientset.CoreV1().Secrets(restoreFromNamespace).Get(ctx,
 		fmt.Sprintf(util.BackrestRepoSecretName, restoreClusterName), metav1.GetOptions{})
 	if err != nil {
-		return fmt.Errorf("Unable to find secret %s for cluster %s, cannot restore from the "+
-			"specified data source",
-			fmt.Sprintf(util.BackrestRepoSecretName, restoreClusterName), restoreClusterName)
+		return fmt.Errorf("Unable to find secret %s for cluster %s (namespace %s), cannot restore "+
+			"from the specified data source",
+			fmt.Sprintf(util.BackrestRepoSecretName, restoreClusterName),
+			restoreClusterName, restoreFromNamespace)
 	}
 
 	// next perform general validation of the restore options
@@ -2377,28 +2386,28 @@ func validateDataSourceParms(request *msgs.CreateClusterRequest) error {
 	// settings are present in backrest repo secret for the backup being restored from
 	s3Restore := backrest.S3RepoTypeCLIOptionExists(restoreOpts)
 	if s3Restore && isMissingExistingDataSourceS3Config(backrestRepoSecret) {
-		return fmt.Errorf("Secret %s is missing the S3 configuration required to restore "+
-			"from an S3 repository", backrestRepoSecret.GetName())
+		return fmt.Errorf("Secret %s (namespace %s) is missing the S3 configuration required to "+
+			"restore from an S3 repository", backrestRepoSecret.GetName(), restoreFromNamespace)
 	}
 
 	// finally, verify that the cluster being restored from is in the proper status, and that no
 	// other clusters currently being bootstrapping from the same cluster
-	clusterList, err := apiserver.Clientset.CrunchydataV1().Pgclusters(namespace).List(ctx, metav1.ListOptions{})
+	clusterList, err := apiserver.Clientset.CrunchydataV1().Pgclusters(restoreFromNamespace).List(ctx, metav1.ListOptions{})
 	if err != nil {
 		return fmt.Errorf("%s: %w", ErrInvalidDataSource, err)
 	}
 	for _, cl := range clusterList.Items {
 
 		if cl.GetName() == restoreClusterName &&
 			cl.Status.State == crv1.PgclusterStateShutdown {
-			return fmt.Errorf("Unable to restore from cluster %s because it has a %s "+
-				"status", restoreClusterName, string(cl.Status.State))
+			return fmt.Errorf("Unable to restore from cluster %s (namespace %s) because it has "+
+				"a %s status", restoreClusterName, restoreFromNamespace, string(cl.Status.State))
 		}
 
 		if cl.Spec.PGDataSource.RestoreFrom == restoreClusterName &&
 			cl.Status.State == crv1.PgclusterStateBootstrapping {
-			return fmt.Errorf("Cluster %s is currently bootstrapping from cluster %s, please "+
-				"try again once it is completes", cl.GetName(), restoreClusterName)
+			return fmt.Errorf("Cluster %s (namespace %s) is currently bootstrapping from cluster %s, please "+
+				"try again once it is completes", cl.GetName(), cl.GetNamespace(), restoreClusterName)
 		}
 	}
 

internal/config/labels.go

@@ -177,11 +177,12 @@ const (
 const GLOBAL_CUSTOM_CONFIGMAP = "pgo-custom-pg-config"
 
 const (
-	LABEL_PGHA_SCOPE        = "crunchy-pgha-scope"
-	LABEL_PGHA_CONFIGMAP    = "pgha-config"
-	LABEL_PGHA_BACKUP_TYPE  = "pgha-backup-type"
-	LABEL_PGHA_ROLE         = "role"
-	LABEL_PGHA_ROLE_PRIMARY = "master"
-	LABEL_PGHA_ROLE_REPLICA = "replica"
-	LABEL_PGHA_BOOTSTRAP    = "pgha-bootstrap"
+	LABEL_PGHA_SCOPE               = "crunchy-pgha-scope"
+	LABEL_PGHA_CONFIGMAP           = "pgha-config"
+	LABEL_PGHA_BACKUP_TYPE         = "pgha-backup-type"
+	LABEL_PGHA_ROLE                = "role"
+	LABEL_PGHA_ROLE_PRIMARY        = "master"
+	LABEL_PGHA_ROLE_REPLICA        = "replica"
+	LABEL_PGHA_BOOTSTRAP           = "pgha-bootstrap"
+	LABEL_PGHA_BOOTSTRAP_NAMESPACE = "pgha-bootstrap-namespace"
 )

internal/controller/job/bootstraphandler.go

@@ -22,7 +22,9 @@ import (
 	"fmt"
 
 	"github.com/crunchydata/postgres-operator/internal/config"
+	"github.com/crunchydata/postgres-operator/internal/operator"
 	backrestoperator "github.com/crunchydata/postgres-operator/internal/operator/backrest"
+	cl "github.com/crunchydata/postgres-operator/internal/operator/cluster"
 	"github.com/crunchydata/postgres-operator/internal/util"
 	crv1 "github.com/crunchydata/postgres-operator/pkg/apis/crunchydata.com/v1"
 
@@ -130,20 +132,22 @@ func (c *Controller) cleanupBootstrapResources(job *apiv1.Job, cluster *crv1.Pgc
 	restore bool) error {
 	ctx := context.TODO()
 
-	namespace := job.GetNamespace()
 	var restoreClusterName string
 	var repoName string
 
+	// get the proper namespace for the bootstrap repo
+	restoreFromNamespace := operator.GetBootstrapNamespace(cluster)
+
 	// clean the repo if a restore, or if a "bootstrap" repo
 	var cleanRepo bool
 	if restore {
 		restoreClusterName = job.GetLabels()[config.LABEL_PG_CLUSTER]
 		repoName = fmt.Sprintf(util.BackrestRepoDeploymentName, restoreClusterName)
-		cleanRepo = true
 	} else {
 		restoreClusterName = cluster.Spec.PGDataSource.RestoreFrom
 		repoName = fmt.Sprintf(util.BackrestRepoDeploymentName, restoreClusterName)
-		repoDeployment, err := c.Client.AppsV1().Deployments(namespace).
+
+		repoDeployment, err := c.Client.AppsV1().Deployments(restoreFromNamespace).
 			Get(ctx, repoName, metav1.GetOptions{})
 		if err != nil {
 			return err
@@ -155,18 +159,25 @@ func (c *Controller) cleanupBootstrapResources(job *apiv1.Job, cluster *crv1.Pgc
 
 	if cleanRepo {
 		// now delete the service for the bootstrap repo
-		if err := c.Client.CoreV1().Services(namespace).Delete(ctx,
+		if err := c.Client.CoreV1().Services(restoreFromNamespace).Delete(ctx,
 			fmt.Sprintf(util.BackrestRepoServiceName, restoreClusterName),
 			metav1.DeleteOptions{}); err != nil && !kerrors.IsNotFound(err) {
 			return err
 		}
 
-		// and finally delete the bootstrap repo deployment
-		if err := c.Client.AppsV1().Deployments(namespace).Delete(ctx, repoName,
+		// and now delete the bootstrap repo deployment
+		if err := c.Client.AppsV1().Deployments(restoreFromNamespace).Delete(ctx, repoName,
 			metav1.DeleteOptions{}); err != nil && !kerrors.IsNotFound(err) {
 			return err
 		}
 	}
 
+	// delete the "bootstrap" version of pgBackRest repo Secret
+	if err := c.Client.CoreV1().Secrets(job.GetNamespace()).Delete(ctx,
+		fmt.Sprintf(cl.BoostrapConfigPrefix, cluster.GetName(), config.LABEL_BACKREST_REPO_SECRET),
+		metav1.DeleteOptions{}); err != nil && !kerrors.IsNotFound(err) {
+		return err
+	}
+
 	return nil
 }

internal/controller/pod/podcontroller.go

@@ -80,7 +80,7 @@ func (c *Controller) onUpdate(oldObj, newObj interface{}) {
 		return
 	}
 
-	var clusterName string
+	var clusterName, namespace string
 	bootstrapCluster := newPodLabels[config.LABEL_PGHA_BOOTSTRAP]
 	// Lookup the pgcluster CR for PG cluster associated with this Pod.  Typically we will use the
 	// 'pg-cluster' label, but if a bootstrap pod we use the 'pgha-bootstrap' label.
@@ -89,8 +89,16 @@ func (c *Controller) onUpdate(oldObj, newObj interface{}) {
 	} else {
 		clusterName = newPodLabels[config.LABEL_PG_CLUSTER]
 	}
-	namespace := newPod.ObjectMeta.Namespace
-	cluster, err := c.Client.CrunchydataV1().Pgclusters(namespace).Get(ctx, clusterName, metav1.GetOptions{})
+	// get the proper namespace for the pgcluster, which could be different than the pod if
+	// restoring across namespaces
+	bootstrapNamespace := newPodLabels[config.LABEL_PGHA_BOOTSTRAP_NAMESPACE]
+	if bootstrapNamespace != "" {
+		namespace = bootstrapNamespace
+	} else {
+		namespace = newPod.ObjectMeta.Namespace
+	}
+	cluster, err := c.Client.CrunchydataV1().Pgclusters(namespace).Get(ctx, clusterName,
+		metav1.GetOptions{})
 	if err != nil {
 		log.Error(err.Error())
 		return

internal/operator/backrest/repo.go

@@ -65,6 +65,7 @@ type RepoDeploymentTemplateFields struct {
 	PodAntiAffinityLabelValue string
 	Replicas                  int
 	BootstrapCluster          string
+	BootstrapNamespace        string
 	Tolerations               string
 }
 
@@ -75,20 +76,23 @@ type RepoServiceTemplateFields struct {
 }
 
 // CreateRepoDeployment creates a pgBackRest repository deployment for a PostgreSQL cluster,
-// while also creating the associated Service and PersistentVolumeClaim.
+// while also creating the associated Service and PersistentVolumeClaim.  Namespace is provided
+// as a parameter since is could vary depending on why the repo is being deployed (e.g. for
+// a new cluster, or to bootstrap a new cluster using the backups from a former PG cluster, which
+// could be in a different namespace).
 func CreateRepoDeployment(clientset kubernetes.Interface, cluster *crv1.Pgcluster,
-	createPVC, bootstrapRepo bool, replicas int) error {
+	createPVC, bootstrapRepo bool, replicas int, namespace string) error {
 	ctx := context.TODO()
 
-	namespace := cluster.GetNamespace()
 	restoreClusterName := cluster.Spec.PGDataSource.RestoreFrom
 
 	repoFields := getRepoDeploymentFields(clientset, cluster, replicas)
 
 	var repoName, serviceName string
 	// if this is a bootstrap repository then we now override certain fields as needed
 	if bootstrapRepo {
-		if err := setBootstrapRepoOverrides(clientset, cluster, repoFields); err != nil {
+		if err := setBootstrapRepoOverrides(clientset, cluster, repoFields,
+			namespace); err != nil {
 			return err
 		}
 		repoName = fmt.Sprintf(util.BackrestRepoPVCName, restoreClusterName)
@@ -114,11 +118,13 @@ func CreateRepoDeployment(clientset kubernetes.Interface, cluster *crv1.Pgcluste
 	// if createPVC is set to true, attempt to create the PVC
 	if createPVC {
 		// create backrest repo PVC with same name as repoName
-		_, err := clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, repoName, metav1.GetOptions{})
+		_, err := clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx,
+			repoName, metav1.GetOptions{})
 		if err == nil {
 			log.Debugf("pvc [%s] already present, will not recreate", repoName)
 		} else if kerrors.IsNotFound(err) {
-			_, err = pvc.CreatePVC(clientset, &cluster.Spec.BackrestStorage, repoName, cluster.Name, namespace)
+			_, err = pvc.CreatePVC(clientset, &cluster.Spec.BackrestStorage, repoName,
+				cluster.Name, namespace)
 			if err != nil {
 				return err
 			}
@@ -152,7 +158,8 @@ func CreateRepoDeployment(clientset kubernetes.Interface, cluster *crv1.Pgcluste
 	operator.SetContainerImageOverride(config.CONTAINER_IMAGE_PGO_BACKREST_REPO,
 		&deployment.Spec.Template.Spec.Containers[0])
 
-	if _, err := clientset.AppsV1().Deployments(namespace).Create(ctx, &deployment, metav1.CreateOptions{}); err != nil &&
+	if _, err := clientset.AppsV1().Deployments(namespace).Create(ctx, &deployment,
+		metav1.CreateOptions{}); err != nil &&
 		!kerrors.IsAlreadyExists(err) {
 		return err
 	}
@@ -174,18 +181,22 @@ func CreateRepoSecret(clientset kubernetes.Interface, cluster *crv1.Pgcluster) e
 	return err
 }
 
-// setBootstrapRepoOverrides overrides certain fields used to populate the pgBackRest repository template
-// as needed to support the creation of a bootstrap repository need to bootstrap a new cluster from an
-// existing data source.
+// setBootstrapRepoOverrides overrides certain fields in the pgBackRest repository template as
+// as needed to support the creation of a bootstrap repository for bootstrapping a new cluster from
+// an existing data source.  The namespace provided as a parameter corresponds to the namespace
+// containing the cluster that will be utilized as the data source for the new pgcluster (and is
+// therefore utilized to obtain any required resources from that cluster, e.g. its pgBackRest
+// configuration).  This namespace could differ from the namespace for the new pgcluster if
+// bootstrapping/restoring from a cluster in another namespace.
 func setBootstrapRepoOverrides(clientset kubernetes.Interface, cluster *crv1.Pgcluster,
-	repoFields *RepoDeploymentTemplateFields) error {
+	repoFields *RepoDeploymentTemplateFields, namespace string) error {
 	ctx := context.TODO()
 
 	restoreClusterName := cluster.Spec.PGDataSource.RestoreFrom
-	namespace := cluster.GetNamespace()
 
 	repoFields.ClusterName = restoreClusterName
 	repoFields.BootstrapCluster = cluster.GetName()
+	repoFields.BootstrapNamespace = cluster.GetNamespace()
 	repoFields.Name = fmt.Sprintf(util.BackrestRepoServiceName, restoreClusterName)
 	repoFields.SshdSecretsName = fmt.Sprintf(util.BackrestRepoSecretName, restoreClusterName)