Backup e2e

hack/teste2e.sh

@@ -0,0 +1,13 @@
+export KUBECONFIG=~/.kube/config
+export KUBERNETES_SERVICE_HOST= 
+export KUBERNETES_SERVICE_PORT=6443
+export S3ENDPOINT=
+export S3ACCESSKEY=
+export S3SECRETKEY=
+export ACK_GINKGO_DEPRECATIONS=1.16.5
+kubectl delete namespace radondb-mysql-e2e
+kubectl get clusterrole|grep mysql|awk  '{print "kubectl delete clusterrole "$1}'|bash
+kubectl get clusterrolebindings|grep mysql|awk  '{print "kubectl delete clusterrolebindings "$1}'|bash
+kubectl get crd|grep mysql|awk  '{print "kubectl delete crd "$1}'|bash
+
+make e2e-local

test/e2e/backup/backup.go

@@ -0,0 +1,177 @@
+/*
+Copyright 2021 RadonDB.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package backup
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"regexp"
+	"strings"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	apiv1alpha1 "github.com/radondb/radondb-mysql-kubernetes/api/v1alpha1"
+	"github.com/radondb/radondb-mysql-kubernetes/test/e2e/framework"
+	"github.com/radondb/radondb-mysql-kubernetes/utils"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+var _ = Describe("MySQL Backup E2E Tests", func() {
+	f := framework.NewFramework("mcbackup-1")
+	_ = f
+
+	var (
+		cluster      *apiv1alpha1.MysqlCluster
+		clusterKey   types.NamespacedName
+		name         string
+		backupSecret *corev1.Secret
+		//timeout      time.Duration
+		POLLING   = 2 * time.Second
+		backupDir string
+		leader    string
+		follower  string
+	)
+
+	BeforeEach(func() {
+		// be careful, mysql allowed hostname lenght is <63
+		name = fmt.Sprintf("bk-%d", rand.Int31()/1000)
+
+		//timeout = 350 * time.Second
+
+		By("create a new backup secret")
+		backupSecret = f.NewBackupSecret()
+		Expect(f.Client.Create(context.TODO(), backupSecret)).To(Succeed(), "create backup secret failed")
+		By("creating a new cluster")
+		cluster = framework.NewCluster(name, f.Namespace.Name)
+		clusterKey = types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}
+		cluster.Spec.BackupSecretName = backupSecret.Name
+		Expect(f.Client.Create(context.TODO(), cluster)).To(Succeed(),
+			"failed to create cluster '%s'", cluster.Name)
+		By("waiting the cluster readiness")
+		framework.WaitClusterReadiness(f, cluster)
+		//get leader
+		for _, node := range cluster.Status.Nodes {
+			if node.RaftStatus.Role == string(utils.Leader) {
+				leader = strings.Split(node.Name, ".")[0]
+			} else if node.RaftStatus.Role == string(utils.Follower) {
+				follower = strings.Split(node.Name, ".")[0]
+			}
+		}
+
+		Expect(f.Client.Get(context.TODO(), clusterKey, cluster)).To(Succeed(), "failed to get cluster %s", cluster.Name)
+
+		Eventually(f.RefreshClusterFn(cluster), f.Timeout, POLLING).Should(
+			framework.HaveClusterReplicas(2))
+		Eventually(f.RefreshClusterFn(cluster), f.Timeout, POLLING).Should(
+			framework.HaveClusterCond(apiv1alpha1.ConditionReady, corev1.ConditionTrue))
+
+		// refresh cluster
+		Expect(f.Client.Get(context.TODO(), clusterKey, cluster)).To(Succeed(),
+			"failed to get cluster %s", cluster.Name)
+
+	})
+
+	It("backup to object store", func() {
+		//exectute sql command in mysql pod
+		By("executing insert data")
+		_, err := f.ExecSQLOnNode(*cluster, leader, "create table testtable (id int)")
+		Expect(err).To(BeNil())
+		_, err = f.ExecSQLOnNode(*cluster, leader, "insert into testtable values (1),(2),(3)")
+		Expect(err).To(BeNil())
+		rows, err := f.ExecSQLOnNode(*cluster, leader, "select * from testtable")
+		Expect(err).To(BeNil(), "failed to execute sql")
+		defer rows.Close()
+		var id int
+		ids := make([]int, 0)
+		for rows.Next() {
+			if err := rows.Scan(&id); err != nil {
+				Fail(err.Error())
+			}
+			ids = append(ids, id)
+		}
+		Expect(ids).To(Equal([]int{1, 2, 3}))
+		Eventually(func() []int {
+			rows, _ :=
+				f.ExecSQLOnNode(*cluster, follower, "select * from testtable ")
+			if rows == nil {
+				return nil
+			}
+			defer rows.Close()
+			var id int
+			ids := make([]int, 0)
+			for rows.Next() {
+				if err := rows.Scan(&id); err != nil {
+					Fail(err.Error())
+				}
+				ids = append(ids, id)
+			}
+			return ids
+		}, f.Timeout, POLLING).Should(Equal([]int{1, 2, 3}))
+		By("executing a backup ")
+		// do the backup
+		backup := framework.NewBackup(cluster, leader)
+		Expect(f.Client.Create(context.TODO(), backup)).To(Succeed(),
+			"failed to create backup '%s'", backup.Name)
+
+		Eventually(f.RefreshBackupFn(backup), f.Timeout, POLLING).Should(
+			framework.HaveBackupComplete())
+
+		if str, err := framework.GetPodLogs(f.ClientSet, f.Namespace.Name, leader, "backup"); err == nil {
+			r, _ := regexp.Compile("backup_[0-9]+")
+			backupDir = r.FindString(str)
+		}
+		nameRestore := fmt.Sprintf("rs-%d", rand.Int31()/1000)
+		By("creating a new cluster from backup")
+		clusterRestore := framework.NewCluster(nameRestore, f.Namespace.Name)
+		clusterKeyRestore := types.NamespacedName{Name: clusterRestore.Name, Namespace: clusterRestore.Namespace}
+		clusterRestore.Spec.BackupSecretName = backupSecret.Name
+		clusterRestore.Spec.RestoreFrom = backupDir
+		Expect(f.Client.Create(context.TODO(), clusterRestore)).To(Succeed(),
+			"failed to create clusterRestore '%s'", clusterRestore.Name)
+		By("waiting the clusterRestore readiness")
+		framework.WaitClusterReadiness(f, clusterRestore)
+		Eventually(f.RefreshClusterFn(clusterRestore), f.Timeout, POLLING).Should(
+			framework.HaveClusterReplicas(2))
+		Eventually(f.RefreshClusterFn(clusterRestore), f.Timeout, POLLING).Should(
+			framework.HaveClusterCond(apiv1alpha1.ConditionReady, corev1.ConditionTrue))
+		Eventually(func() []int {
+			rows, _ := f.ExecSQLOnNode(*clusterRestore, fmt.Sprintf("%s-mysql-0", nameRestore), "select * from testtable ")
+			if rows == nil {
+				return nil
+			}
+			defer rows.Close()
+			var id int
+			ids := make([]int, 0)
+			for rows.Next() {
+				if err := rows.Scan(&id); err != nil {
+					Fail(err.Error())
+				}
+				ids = append(ids, id)
+			}
+			return ids
+		}, f.Timeout, POLLING).Should(Equal([]int{1, 2, 3}))
+
+		// refresh clusterRestore
+		Expect(f.Client.Get(context.TODO(), clusterKeyRestore, clusterRestore)).To(Succeed(),
+			"failed to get clusterRestore %s", clusterRestore.Name)
+		Expect(f.Client.Delete(context.TODO(), clusterRestore)).To(Succeed(),
+			"failed to delete clusterRestore '%s'", clusterRestore.Name)
+	})
+
+})

test/e2e/cluster/cluster.go

@@ -0,0 +1,85 @@
+/*
+Copyright 2021 RadonDB.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"k8s.io/apimachinery/pkg/types"
+
+	apiv1alpha1 "github.com/radondb/radondb-mysql-kubernetes/api/v1alpha1"
+	"github.com/radondb/radondb-mysql-kubernetes/test/e2e/framework"
+)
+
+const (
+	POLLING = 2 * time.Second
+)
+
+var _ = Describe("MySQL Cluster E2E Tests", func() {
+	f := framework.NewFramework("mc-1")
+	two := int32(2)
+	three := int32(3)
+	five := int32(5)
+
+	var (
+		cluster    *apiv1alpha1.MysqlCluster
+		clusterKey types.NamespacedName
+		name       string
+	)
+
+	BeforeEach(func() {
+		// Be careful, mysql allowed hostname lenght is <63.
+		name = fmt.Sprintf("cl-%d", rand.Int31()/1000)
+
+		By("creating a new cluster")
+		cluster = framework.NewCluster(name, f.Namespace.Name)
+		clusterKey = types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}
+		Expect(f.Client.Create(context.TODO(), cluster)).To(Succeed(), "failed to create cluster '%s'", cluster.Name)
+
+		By("testing the cluster readiness")
+		framework.WaitClusterReadiness(f, cluster)
+		Expect(f.Client.Get(context.TODO(), clusterKey, cluster)).To(Succeed(), "failed to get cluster %s", cluster.Name)
+	})
+
+	It("scale out/in a cluster, 2 -> 3 -> 5 -> 3 -> 2", func() {
+		By("test cluster is ready after scale out 2 -> 3")
+		cluster.Spec.Replicas = &three
+		Expect(f.Client.Update(context.TODO(), cluster)).To(Succeed())
+		fmt.Println("scale time: ", framework.WaitClusterReadiness(f, cluster))
+
+		By("test cluster is ready after scale out 3 -> 5")
+		cluster.Spec.Replicas = &five
+		Expect(f.Client.Update(context.TODO(), cluster)).To(Succeed())
+		fmt.Println("scale time: ", framework.WaitClusterReadiness(f, cluster))
+
+		By("test cluster is ready after scale in 5 -> 3")
+		cluster.Spec.Replicas = &three
+		Expect(f.Client.Update(context.TODO(), cluster)).To(Succeed())
+		fmt.Println("scale time: ", framework.WaitClusterReadiness(f, cluster))
+
+		By("test cluster is ready after scale in 3 -> 2")
+		cluster.Spec.Replicas = &two
+		Expect(f.Client.Update(context.TODO(), cluster)).To(Succeed())
+		fmt.Println("scale time: ", framework.WaitClusterReadiness(f, cluster))
+	})
+
+})

test/e2e/e2e.go

@@ -17,30 +17,61 @@ limitations under the License.
 package e2e
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"path"
+	"strings"
 	"testing"
 
 	"github.com/golang/glog"
 	. "github.com/onsi/ginkgo"
 	"github.com/onsi/ginkgo/config"
 	"github.com/onsi/ginkgo/reporters"
 	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	runtimeutils "k8s.io/apimachinery/pkg/util/runtime"
+	clientset "k8s.io/client-go/kubernetes"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/radondb/radondb-mysql-kubernetes/test/e2e/framework"
 	"github.com/radondb/radondb-mysql-kubernetes/test/e2e/framework/ginkgowrapper"
 )
 
 const (
-	operatorNamespace = "mysql-operator"
+	RadondbMysqlE2eNamespace = "radondb-mysql-e2e"
 )
 
+var releaseName = framework.RandStr(6)
+
 var _ = SynchronizedBeforeSuite(func() []byte {
-	// BeforeSuite logic.
+	kubeCfg, err := framework.LoadConfig()
+	Expect(err).To(Succeed())
+	// restClient := core.NewForConfigOrDie(kubeCfg).RESTClient()
+
+	c, err := client.New(kubeCfg, client.Options{})
+	if err != nil {
+		Fail(fmt.Sprintf("can't instantiate k8s client: %s", err))
+	}
+
+	// ginkgo node 1
+	By("Install operator")
+	operatorNsObj := &corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: RadondbMysqlE2eNamespace,
+		},
+	}
+	if err := c.Create(context.TODO(), operatorNsObj); err != nil {
+		if !strings.Contains(err.Error(), "already exists") {
+			Fail(fmt.Sprintf("can't create mysql-operator namespace: %s", err))
+		}
+	}
+	framework.HelmInstallChart(releaseName, RadondbMysqlE2eNamespace)
+
 	return nil
+
 }, func(data []byte) {
 	// all other nodes
 	framework.Logf("Running BeforeSuite actions on all node")
@@ -50,7 +81,25 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 // Here, the order of functions is reversed; first, the function which runs everywhere,
 // and then the function that only runs on the first Ginkgo node.
 var _ = SynchronizedAfterSuite(func() {
-	// AfterSuite logic.
+	// Run on all Ginkgo nodes
+	framework.Logf("Running AfterSuite actions on all node")
+	framework.RunCleanupActions()
+
+	// get the kubernetes client
+	kubeCfg, err := framework.LoadConfig()
+	Expect(err).To(Succeed())
+
+	client, err := clientset.NewForConfig(kubeCfg)
+	Expect(err).NotTo(HaveOccurred())
+
+	By("Remove operator release")
+	framework.HelmPurgeRelease(releaseName, RadondbMysqlE2eNamespace)
+
+	By("Delete operator namespace")
+
+	if err := framework.DeleteNS(client, RadondbMysqlE2eNamespace, framework.DefaultNamespaceDeletionTimeout); err != nil {
+		framework.Failf(fmt.Sprintf("Can't delete namespace: %s", err))
+	}
 }, func() {
 	// Run only Ginkgo on node 1
 	framework.Logf("Running AfterSuite actions on node 1")
@@ -84,13 +133,13 @@ func RunE2ETests(t *testing.T) {
 
 			// add logs dumper
 			if framework.TestContext.DumpLogsOnFailure {
-				rps = append(rps, NewLogsPodReporter(operatorNamespace, path.Join(framework.TestContext.ReportDir,
+				rps = append(rps, NewLogsPodReporter(RadondbMysqlE2eNamespace, path.Join(framework.TestContext.ReportDir,
 					fmt.Sprintf("pods_logs_%d_%d.txt", config.GinkgoConfig.RandomSeed, config.GinkgoConfig.ParallelNode))))
 			}
 		} else {
 			// if reportDir is not specified then print logs to stdout
 			if framework.TestContext.DumpLogsOnFailure {
-				rps = append(rps, NewLogsPodReporter(operatorNamespace, ""))
+				rps = append(rps, NewLogsPodReporter(RadondbMysqlE2eNamespace, ""))
 			}
 		}
 		return

test/e2e/e2e_test.go

@@ -19,8 +19,10 @@ package e2e
 import (
 	"testing"
 
+	// _ "github.com/radondb/radondb-mysql-kubernetes/test/e2e/cluster"
+	_ "github.com/radondb/radondb-mysql-kubernetes/test/e2e/backup"
 	"github.com/radondb/radondb-mysql-kubernetes/test/e2e/framework"
-	_ "github.com/radondb/radondb-mysql-kubernetes/test/e2e/simplecase"
+	// _ "github.com/radondb/radondb-mysql-kubernetes/test/e2e/simplecase"
 )
 
 func init() {