tfo unlock

README.md

@@ -51,4 +51,4 @@ The server will be accessible at `http://localhost:3000` (or the specified port)
 
 ## Testing
 
-Use tools like `curl` or Postman to test the API endpoints. Verify that the responses match your expectations.
+Use tools like `curl` or Postman to test the API endpoints. Verify that the responses match your expectations.

go.mod

@@ -3,8 +3,8 @@ module github.com/galleybytes/terraform-operator-api
 go 1.19
 
 require (
+	github.com/akyoto/cache v1.0.6
 	github.com/galleybytes/terraform-operator v0.14.0
-	github.com/gammazero/deque v0.2.1
 	github.com/gin-gonic/gin v1.8.1
 	github.com/golang-jwt/jwt/v4 v4.4.3
 	github.com/isaaguilar/kedge v0.0.0-20230623005919-25931c711d84
@@ -19,7 +19,6 @@ require (
 require (
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/MakeNowJust/heredoc v1.0.0 // indirect
-	github.com/akyoto/cache v1.0.6 // indirect
 	github.com/beevik/etree v1.1.0 // indirect
 	github.com/chai2010/gettext-go v1.0.2 // indirect
 	github.com/crewjam/httperr v0.2.0 // indirect
@@ -40,7 +39,6 @@ require (
 	github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/russellhaering/goxmldsig v1.2.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/spf13/cobra v1.6.0 // indirect
@@ -96,7 +94,6 @@ require (
 )
 
 require (
-	github.com/DATA-DOG/go-sqlmock v1.5.0
 	github.com/creack/pty v1.1.18
 	github.com/crewjam/saml v0.4.13
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
@@ -131,7 +128,7 @@ require (
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5
-	github.com/stretchr/testify v1.8.4
+	github.com/stretchr/testify v1.8.4 // indirect
 	github.com/subosito/gotenv v1.4.1 // indirect
 	github.com/ucarion/saml v0.1.2
 	github.com/ugorji/go/codec v1.2.7 // indirect

go.sum

@@ -40,8 +40,6 @@ github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOEl
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
-github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
 github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ=
 github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE=
 github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
@@ -115,12 +113,8 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
 github.com/fvbommel/sortorder v1.0.1 h1:dSnXLt4mJYH25uDDGa3biZNQsozaUWDSWeKJ0qqFfzE=
 github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
-github.com/galleybytes/terraform-operator v0.12.1 h1:sqLZtnxlNgIl0psnBjnjwQ676P+q2NIhcoUfuGcHyYY=
-github.com/galleybytes/terraform-operator v0.12.1/go.mod h1:UBmC5dPK2dBA09AjLNW4szw5VqQ8mndXrR8Gp97GRtE=
 github.com/galleybytes/terraform-operator v0.14.0 h1:fmknK+CyaG12Oz0wE0Oq72zpnRkzx0zMwcLCF/am0Vs=
 github.com/galleybytes/terraform-operator v0.14.0/go.mod h1:UBmC5dPK2dBA09AjLNW4szw5VqQ8mndXrR8Gp97GRtE=
-github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0=
-github.com/gammazero/deque v0.2.1/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU=
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
@@ -455,7 +449,6 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs=
@@ -867,8 +860,6 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gorm.io/driver/postgres v1.3.9 h1:lWGiVt5CijhQAg0PWB7Od1RNcBw/jS4d2cAScBcSDXg=
 gorm.io/driver/postgres v1.3.9/go.mod h1:qw/FeqjxmYqW5dBcYNBsnhQULIApQdk7YuuDPktVi1U=
 gorm.io/gorm v1.23.7/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
-gorm.io/gorm v1.23.8 h1:h8sGJ+biDgBA1AD1Ha9gFCx7h8npU7AsLdlkX0n2TpE=
-gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
 gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls=
 gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=

pkg/api/api.go

@@ -112,6 +112,7 @@ func (h APIHandler) RegisterRoutes() {
 	cluster.GET("/:cluster_name/resource/:namespace/:name/poll", h.ResourcePoll) // Poll for resource objects in the cluster
 	cluster.GET("/:cluster_name/resource/:namespace/:name/debug", h.Debugger)
 	cluster.GET("/:cluster_name/debug/:namespace/:name", h.Debugger) // Alias
+	cluster.GET("/:cluster_name/resource/:namespace/:name/unlock", h.UnlockTerraform)
 	cluster.GET("/:cluster_name/resource/:namespace/:name/status", h.ResourceStatusCheck)
 	cluster.GET("/:cluster_name/status/:namespace/:name", h.ResourceStatusCheck) // Alias
 	cluster.GET("/:cluster_name/resource/:namespace/:name/last-task-log", h.LastTaskLog)

pkg/api/get_record.go

@@ -30,6 +30,7 @@ import (
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/cli-runtime/pkg/genericclioptions"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/remotecommand"
 	"k8s.io/kubectl/pkg/cmd/exec"
 	"k8s.io/kubectl/pkg/scheme"
@@ -732,7 +733,20 @@ func (h APIHandler) Debugger(c *gin.Context) {
 	}
 	defer conn.Close()
 
-	podExecReadWriter, err := New(h.clientset, clusterName, namespace, name, c, cmd)
+	execCommand := []string{
+		"/bin/bash",
+		"-c",
+		`cd $TFO_MAIN_MODULE && \
+			export PS1="\\w\\$ " && \
+			if [[ -n "$AWS_WEB_IDENTITY_TOKEN_FILE" ]]; then
+				export $(irsa-tokengen);
+				echo printf "\nAWS creds set from token file\n"
+			fi && \
+			printf "\nTry running 'terraform init'\n\n" && bash
+		`,
+	}
+
+	podExecReadWriter, err := newSessionInTerraformDebugPod(h.clientset, clusterName, namespace, name, c, cmd, execCommand)
 	if err != nil {
 		log.Printf("Failed to connect to debug pod: %s", err)
 		return
@@ -781,6 +795,51 @@ func (h APIHandler) Debugger(c *gin.Context) {
 	// return
 }
 
+func (h APIHandler) UnlockTerraform(c *gin.Context) {
+	clusterName := c.Param("cluster_name")
+	clusterID := h.getClusterID(clusterName)
+	if clusterID == 0 {
+		c.JSON(http.StatusUnprocessableEntity, response(http.StatusUnprocessableEntity, fmt.Sprintf("cluster_name '%s' not found", clusterName), nil))
+		return
+	}
+	name := c.Param("name")
+	namespace := c.Param("namespace")
+	if _, err := getResource(h.clientset, clusterName, namespace, name, c); err != nil {
+		c.JSON(http.StatusUnprocessableEntity, response(http.StatusUnprocessableEntity, fmt.Sprintf("tf resource '%s/%s' not found", namespace, name), nil))
+		return
+	}
+
+	command := []string{
+		"/bin/bash",
+		"-c",
+		`cd $TFO_MAIN_MODULE && \
+		file=$(mktemp) && \
+		terraform plan -no-color 2>$file
+		if [[ ! -s "$file" ]] ; then
+          echo "\nno lock detected exiting"
+          exit 0
+		fi && \
+		cat $file
+		lock=$(grep -A1 "Lock Info" $file | grep "ID") && \
+		lock_id=$(echo $lock | sed -n 's/.*\([0-9a-fA-F-]\{36\}\).*/\1/p') && \
+		echo lock=$lock && \
+		echo lock_id=$lock_id && \
+		if [ -n "$lock_id" ]; then
+		terraform force-unlock -force $lock_id
+		fi && \
+		echo "Done"`,
+	}
+
+	err := runUnlockTerraformDebugPod(h.clientset, clusterName, namespace, name, c, command)
+	if err != nil {
+		c.JSON(http.StatusUnprocessableEntity, response(http.StatusUnprocessableEntity, fmt.Sprintf("terraform unlock failed: %s", err), nil))
+		return
+	}
+
+	c.JSON(http.StatusOK, response(http.StatusOK, "terraform unlocked", nil))
+
+}
+
 type wsWrapper struct {
 	*websocket.Conn
 }
@@ -861,7 +920,7 @@ func (t TermSizer) Next() *remotecommand.TerminalSize {
 // }
 
 // command string, argv []string, headers map[string][]string, options ...Option
-func New(clientset kubernetes.Interface, clusterName, namespace, name string, c *gin.Context, cmd []string) (*PodExec, error) {
+func newSessionInTerraformDebugPod(clientset kubernetes.Interface, clusterName, namespace, name string, c *gin.Context, cmd, execCommand []string) (*PodExec, error) {
 	pty, tty, err := ptylib.Open()
 	if err != nil {
 		log.Fatal(err)
@@ -879,7 +938,7 @@ func New(clientset kubernetes.Interface, clusterName, namespace, name string, c
 
 	go func() {
 		defer pty.Close()
-		err := RemoteDebug(clientset, clusterName, namespace, name, tty, c, termSizer, cmd)
+		err := RemoteDebug(clientset, clusterName, namespace, name, tty, c, termSizer, cmd, execCommand)
 		log.Println("Pod exec exited")
 		closeCh <- err
 	}()
@@ -928,32 +987,59 @@ func (p *PodExec) Close() error {
 	return nil
 }
 
-// RemoteDebug starts the debug pod and connects in a tty that will be synced thru a websocket. Anything written to
-// stdout will be synced to the tty. stderr logs will show up in the api logs and not the tty.
-func RemoteDebug(parentClientset kubernetes.Interface, clusterName, namespace, name string, tty *os.File, c *gin.Context, terminalSizeQueue remotecommand.TerminalSizeQueue, cmd []string) error {
+func createDebugPodManifest(c *gin.Context, config *rest.Config, namespace, name string, command []string) (*corev1.Pod, error) {
+	tfoclientset := tfo.NewForConfigOrDie(config)
+	tfclient := tfoclientset.TfV1beta1().Terraforms(namespace)
+	tf, err := tfclient.Get(c, name, metav1.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+	pod := generatePod(tf, command)
+	return pod, nil
+}
 
+func runUnlockTerraformDebugPod(parentClientset kubernetes.Interface, clusterName, namespace, name string, c *gin.Context, command []string) error {
 	config, err := getVclusterConfig(parentClientset, "internal", clusterName)
 	if err != nil {
 		return err
 	}
-	tfoclientset := tfo.NewForConfigOrDie(config)
-	clientset := kubernetes.NewForConfigOrDie(config)
-	tfclient := tfoclientset.TfV1beta1().Terraforms(namespace)
-	// newSession()
 
-	// tfoclientset := session.tfoclientset.TfV1beta1().Terraforms(namespace)
+	pod, err := createDebugPodManifest(c, config, namespace, name, command)
+	if err != nil {
+		return err
+	}
+
+	clientset := kubernetes.NewForConfigOrDie(config)
 	podClient := clientset.CoreV1().Pods(namespace)
+	pod, err = podClient.Create(c, pod, metav1.CreateOptions{})
+	if err != nil {
+		return err
+	}
 
-	tf, err := tfclient.Get(c, name, metav1.GetOptions{})
+	return getPodStatus(pod, clientset, namespace)
+}
+
+// RemoteDebug starts the debug pod and connects in a tty that will be synced thru a websocket. Anything written to
+// stdout will be synced to the tty. stderr logs will show up in the api logs and not the tty.
+func RemoteDebug(parentClientset kubernetes.Interface, clusterName, namespace, name string, tty *os.File, c *gin.Context, terminalSizeQueue remotecommand.TerminalSizeQueue, cmd, execCommand []string) error {
+
+	config, err := getVclusterConfig(parentClientset, "internal", clusterName)
+	if err != nil {
+		return err
+	}
+
+	pod, err := createDebugPodManifest(c, config, namespace, name, nil)
 	if err != nil {
 		return err
 	}
 
-	pod := generatePod(tf)
+	clientset := kubernetes.NewForConfigOrDie(config)
+	podClient := clientset.CoreV1().Pods(namespace)
 	pod, err = podClient.Create(c, pod, metav1.CreateOptions{})
 	if err != nil {
 		return err
 	}
+
 	defer podClient.Delete(c, pod.Name, metav1.DeleteOptions{})
 
 	fmt.Printf("Connecting to %s ", pod.Name)
@@ -1001,18 +1087,6 @@ func RemoteDebug(parentClientset kubernetes.Interface, clusterName, namespace, n
 	}
 	// log.Println(file.Name())
 	// log.Println("Setting up request")
-	execCommand := []string{
-		"/bin/bash",
-		"-c",
-		`cd $TFO_MAIN_MODULE && \
-			export PS1="\\w\\$ " && \
-			if [[ -n "$AWS_WEB_IDENTITY_TOKEN_FILE" ]]; then
-				export $(irsa-tokengen);
-				echo printf "\nAWS creds set from token file\n"
-			fi && \
-			printf "\nTry running 'terraform init'\n\n" && bash
-		`,
-	}
 
 	if len(cmd) > 0 {
 		execCommand = cmd
@@ -1061,6 +1135,60 @@ func RemoteDebug(parentClientset kubernetes.Interface, clusterName, namespace, n
 	return nil
 }
 
+func getPodStatus(pod *corev1.Pod, clientset *kubernetes.Clientset, namespace string) error {
+	podStatusStartTime := time.Now()
+	for {
+		pod, err := clientset.CoreV1().Pods(namespace).Get(context.TODO(), pod.Name, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+		phase := pod.Status.Phase
+		if phase == "Succeeded" {
+			err := deletePod(namespace, pod.Name, clientset)
+			if err != nil {
+				return err
+			}
+			return nil
+		} else if phase == "Failed" {
+			err := deletePod(namespace, pod.Name, clientset)
+			if err != nil {
+				return err
+			}
+			return fmt.Errorf("Pod failed: phase %s", phase)
+		}
+		time.Sleep(5 * time.Second)
+		isDeleted, err := podTimeToLive(podStatusStartTime, pod.Name, namespace, clientset, 300)
+		if err != nil {
+			return err
+		}
+		if isDeleted == true {
+			return fmt.Errorf("Pod did not complete in time and was forcefully deleted")
+		}
+
+	}
+
+}
+
+func deletePod(namespace, podName string, clientset *kubernetes.Clientset) error {
+	err := clientset.CoreV1().Pods(namespace).Delete(context.TODO(), podName, metav1.DeleteOptions{})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func podTimeToLive(podStatusStartTime time.Time, podName, namespace string, clientset *kubernetes.Clientset, timeToLive time.Duration) (bool, error) {
+	var deleteCompleted = false
+	if time.Since(podStatusStartTime) >= timeToLive*time.Second {
+		err := deletePod(namespace, podName, clientset)
+		if err != nil {
+			return deleteCompleted, err
+		}
+		deleteCompleted = true
+	}
+	return deleteCompleted, nil
+}
+
 // func isTerminal(file *os.File) bool {
 
 // 	inFd := file.Fd()
@@ -1070,7 +1198,7 @@ func RemoteDebug(parentClientset kubernetes.Interface, clusterName, namespace, n
 
 // }
 
-func generatePod(tf *tfv1beta1.Terraform) *corev1.Pod {
+func generatePod(tf *tfv1beta1.Terraform, command []string) *corev1.Pod {
 	terraformVersion := tf.Spec.TerraformVersion
 	if terraformVersion == "" {
 		terraformVersion = "1.1.5"
@@ -1230,13 +1358,17 @@ func generatePod(tf *tfv1beta1.Terraform) *corev1.Pod {
 	}
 	restartPolicy := corev1.RestartPolicyNever
 
+	if command == nil {
+		command = []string{
+			"/bin/sleep", "86400",
+		}
+	}
+
 	containers = append(containers, corev1.Container{
 		SecurityContext: securityContext,
 		Name:            "debug",
 		Image:           "ghcr.io/galleybytes/terraform-operator-tftaskv1.1.0:" + terraformVersion,
-		Command: []string{
-			"/bin/sleep", "86400",
-		},
+		Command:         command,
 		ImagePullPolicy: corev1.PullIfNotPresent,
 		EnvFrom:         envFrom,
 		Env:             env,