Concurrent backup

justinrlee · Jan 8, 2018 · baa4349 · baa4349
1 parent 8fc4ca2
commit baa4349
Show file tree

Hide file tree

Showing 3 changed files with 69 additions and 13 deletions.
diff --git a/cmd/backup.go b/cmd/backup.go
@@ -61,20 +61,35 @@ to quickly create a Cobra application.`,
 
 		secretSlice := []Secret{}
 
-		// Get all secrets, add them to the files array
-		for _, secretID := range secrets.Array {
-			fmt.Printf("Getting secret '%s'\n", secretID)
-			// secretValue, err := cluster.Get("/secrets/v1/secret/default/" + secretPath)
-			secretJSON, returnCode, err := cluster.Call("GET", "/secrets/v1/secret/default/"+secretID, nil)
-			if err != nil || returnCode != http.StatusOK {
-				fmt.Println("TODO: error handling here")
-				panic(err)
-			}
-
-			e := encrypt(string(secretJSON), cipherkey)
-			secretSlice = append(secretSlice, Secret{ID: secretID, EncryptedJSON: e})
+		secretChan := make(chan Secret)
+
+		// fmt.Println("Calling go GetSecrets")
+		go cluster.GetSecrets(secrets.Array, cipherkey, secretChan, 10)
+
+		// fmt.Println("Starting to receive")
+		for i := 0; i < len(secrets.Array); i++ {
+			// fmt.Printf("Waiting for %d\n", i)
+			s := <-secretChan
+			secretSlice = append(secretSlice, s)
 		}
 
+		// fmt.Println(secretSlice)
+
+// func (c *Cluster) GetSecrets(secrets []string, cipherKey string, secretChan chan<- Secret, qsize int) {
+		// // Get all secrets, add them to the files array
+		// for _, secretID := range secrets.Array {
+		// 	fmt.Printf("Getting secret '%s'\n", secretID)
+		// 	// secretValue, err := cluster.Get("/secrets/v1/secret/default/" + secretPath)
+		// 	secretJSON, returnCode, err := cluster.Call("GET", "/secrets/v1/secret/default/"+secretID, nil)
+		// 	if err != nil || returnCode != http.StatusOK {
+		// 		fmt.Println("TODO: error handling here")
+		// 		panic(err)
+		// 	}
+
+		// 	e := encrypt(string(secretJSON), cipherkey)
+		// 	secretSlice = append(secretSlice, Secret{ID: secretID, EncryptedJSON: e})
+		// }
+
 		fmt.Println("Writing to tar at " + destfile)
 		writeTar(secretSlice, destfile)
 

diff --git a/cmd/restore.go b/cmd/restore.go
@@ -64,7 +64,7 @@ to quickly create a Cobra application.`,
 
 			resp, code, err := cluster.Call("PUT", secretPath, []byte(plaintext))
 			if code == 201 {
-				fmt.Println("Secret" + item.ID + "successfully created.")
+				fmt.Println("Secret [" + item.ID + "] successfully created.")
 			} else if code == 409 {
 				presp, pcode, perr := cluster.Call("PATCH", secretPath, []byte(plaintext))
 				if pcode == 204 {

diff --git a/cmd/utility-cluster.go b/cmd/utility-cluster.go
@@ -117,3 +117,44 @@ func (c *Cluster) Call(verb string, path string, buf []byte) (body []byte, retur
 
 	return body, resp.StatusCode, err
 }
+
+// Get secret
+func (c *Cluster) GetSecret(secretID string, cipherKey string, secretChan chan<- Secret, queue chan int) {
+		<- queue // Wait for there to be an open spot in the queue
+		fmt.Printf("Getting secret '%s'\n", secretID)
+		secretJSON, returnCode, err := c.Call("GET", "/secrets/v1/secret/default/"+secretID, nil)
+		if err != nil || returnCode != http.StatusOK {
+			fmt.Println("Unable to retrieve secret '%s'\n. [%d]: %s", secretID, returnCode, err.Error)
+			secretChan <- Secret{ID: "", EncryptedJSON: ""}
+			queue <- 0 // Release queue spot
+		} else {
+			e := encrypt(string(secretJSON), cipherKey)
+			secretChan <- Secret{ID: secretID, EncryptedJSON: e}
+			queue <- 0 // Release queue spot
+		}
+}
+
+func (c *Cluster) GetSecrets(secrets []string, cipherKey string, secretChan chan Secret, qsize int) {
+	queue := make(chan int, qsize)
+	for i:= 0; i < qsize; i++ {
+		// fmt.Println("Writing 0 to queue")
+		queue <- 0
+	}
+	// Spins off a bunch of goroutines to get secrets and add them to secretChan.  Should be rate limited by qsize
+	for _, secretID := range secrets {
+		go c.GetSecret(secretID, cipherKey, secretChan, queue)
+	}
+}
+
+// for _, secretID := range secrets.Array {
+// 	fmt.Printf("Getting secret '%s'\n", secretID)
+// 	// secretValue, err := cluster.Get("/secrets/v1/secret/default/" + secretPath)
+// 	secretJSON, returnCode, err := cluster.Call("GET", "/secrets/v1/secret/default/"+secretID, nil)
+// 	if err != nil || returnCode != http.StatusOK {
+// 		fmt.Println("TODO: error handling here")
+// 		panic(err)
+// 	}
+
+// 	e := encrypt(string(secretJSON), cipherkey)
+// 	secretSlice = append(secretSlice, Secret{ID: secretID, EncryptedJSON: e})
+// }