diff --git a/README.md b/README.md index 08d8aef..271442e 100644 --- a/README.md +++ b/README.md @@ -51,4 +51,4 @@ The server will be accessible at `http://localhost:3000` (or the specified port) ## Testing -Use tools like `curl` or Postman to test the API endpoints. Verify that the responses match your expectations. +Use tools like `curl` or Postman to test the API endpoints. Verify that the responses match your expectations. \ No newline at end of file diff --git a/go.mod b/go.mod index 1474f70..10ed5e3 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module github.com/galleybytes/terraform-operator-api go 1.19 require ( + github.com/akyoto/cache v1.0.6 github.com/galleybytes/terraform-operator v0.14.0 - github.com/gammazero/deque v0.2.1 github.com/gin-gonic/gin v1.8.1 github.com/golang-jwt/jwt/v4 v4.4.3 github.com/isaaguilar/kedge v0.0.0-20230623005919-25931c711d84 @@ -19,7 +19,6 @@ require ( require ( github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect - github.com/akyoto/cache v1.0.6 // indirect github.com/beevik/etree v1.1.0 // indirect github.com/chai2010/gettext-go v1.0.2 // indirect github.com/crewjam/httperr v0.2.0 // indirect @@ -40,7 +39,6 @@ require ( github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/russellhaering/goxmldsig v1.2.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spf13/cobra v1.6.0 // indirect @@ -96,7 +94,6 @@ require ( ) require ( - github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/creack/pty v1.1.18 github.com/crewjam/saml v0.4.13 github.com/fsnotify/fsnotify v1.6.0 // indirect @@ -131,7 +128,7 @@ require ( github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.8.4 // indirect github.com/subosito/gotenv v1.4.1 // indirect github.com/ucarion/saml v0.1.2 github.com/ugorji/go/codec v1.2.7 // indirect diff --git a/go.sum b/go.sum index 6f6ab45..259020a 100644 --- a/go.sum +++ b/go.sum @@ -40,8 +40,6 @@ github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOEl github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= -github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= @@ -115,12 +113,8 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/fvbommel/sortorder v1.0.1 h1:dSnXLt4mJYH25uDDGa3biZNQsozaUWDSWeKJ0qqFfzE= github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= -github.com/galleybytes/terraform-operator v0.12.1 h1:sqLZtnxlNgIl0psnBjnjwQ676P+q2NIhcoUfuGcHyYY= -github.com/galleybytes/terraform-operator v0.12.1/go.mod h1:UBmC5dPK2dBA09AjLNW4szw5VqQ8mndXrR8Gp97GRtE= github.com/galleybytes/terraform-operator v0.14.0 h1:fmknK+CyaG12Oz0wE0Oq72zpnRkzx0zMwcLCF/am0Vs= github.com/galleybytes/terraform-operator v0.14.0/go.mod h1:UBmC5dPK2dBA09AjLNW4szw5VqQ8mndXrR8Gp97GRtE= -github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0= -github.com/gammazero/deque v0.2.1/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= @@ -455,7 +449,6 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= @@ -867,8 +860,6 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/driver/postgres v1.3.9 h1:lWGiVt5CijhQAg0PWB7Od1RNcBw/jS4d2cAScBcSDXg= gorm.io/driver/postgres v1.3.9/go.mod h1:qw/FeqjxmYqW5dBcYNBsnhQULIApQdk7YuuDPktVi1U= gorm.io/gorm v1.23.7/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= -gorm.io/gorm v1.23.8 h1:h8sGJ+biDgBA1AD1Ha9gFCx7h8npU7AsLdlkX0n2TpE= -gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls= gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= diff --git a/pkg/api/api.go b/pkg/api/api.go index 84f4a61..56bdf8e 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -112,6 +112,7 @@ func (h APIHandler) RegisterRoutes() { cluster.GET("/:cluster_name/resource/:namespace/:name/poll", h.ResourcePoll) // Poll for resource objects in the cluster cluster.GET("/:cluster_name/resource/:namespace/:name/debug", h.Debugger) cluster.GET("/:cluster_name/debug/:namespace/:name", h.Debugger) // Alias + cluster.GET("/:cluster_name/resource/:namespace/:name/unlock", h.UnlockTerraform) cluster.GET("/:cluster_name/resource/:namespace/:name/status", h.ResourceStatusCheck) cluster.GET("/:cluster_name/status/:namespace/:name", h.ResourceStatusCheck) // Alias cluster.GET("/:cluster_name/resource/:namespace/:name/last-task-log", h.LastTaskLog) diff --git a/pkg/api/get_record.go b/pkg/api/get_record.go index 17f86de..4d140af 100644 --- a/pkg/api/get_record.go +++ b/pkg/api/get_record.go @@ -30,6 +30,7 @@ import ( "k8s.io/apimachinery/pkg/watch" "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/remotecommand" "k8s.io/kubectl/pkg/cmd/exec" "k8s.io/kubectl/pkg/scheme" @@ -732,7 +733,20 @@ func (h APIHandler) Debugger(c *gin.Context) { } defer conn.Close() - podExecReadWriter, err := New(h.clientset, clusterName, namespace, name, c, cmd) + execCommand := []string{ + "/bin/bash", + "-c", + `cd $TFO_MAIN_MODULE && \ + export PS1="\\w\\$ " && \ + if [[ -n "$AWS_WEB_IDENTITY_TOKEN_FILE" ]]; then + export $(irsa-tokengen); + echo printf "\nAWS creds set from token file\n" + fi && \ + printf "\nTry running 'terraform init'\n\n" && bash + `, + } + + podExecReadWriter, err := newSessionInTerraformDebugPod(h.clientset, clusterName, namespace, name, c, cmd, execCommand) if err != nil { log.Printf("Failed to connect to debug pod: %s", err) return @@ -781,6 +795,51 @@ func (h APIHandler) Debugger(c *gin.Context) { // return } +func (h APIHandler) UnlockTerraform(c *gin.Context) { + clusterName := c.Param("cluster_name") + clusterID := h.getClusterID(clusterName) + if clusterID == 0 { + c.JSON(http.StatusUnprocessableEntity, response(http.StatusUnprocessableEntity, fmt.Sprintf("cluster_name '%s' not found", clusterName), nil)) + return + } + name := c.Param("name") + namespace := c.Param("namespace") + if _, err := getResource(h.clientset, clusterName, namespace, name, c); err != nil { + c.JSON(http.StatusUnprocessableEntity, response(http.StatusUnprocessableEntity, fmt.Sprintf("tf resource '%s/%s' not found", namespace, name), nil)) + return + } + + command := []string{ + "/bin/bash", + "-c", + `cd $TFO_MAIN_MODULE && \ + file=$(mktemp) && \ + terraform plan -no-color 2>$file + if [[ ! -s "$file" ]] ; then + echo "\nno lock detected exiting" + exit 0 + fi && \ + cat $file + lock=$(grep -A1 "Lock Info" $file | grep "ID") && \ + lock_id=$(echo $lock | sed -n 's/.*\([0-9a-fA-F-]\{36\}\).*/\1/p') && \ + echo lock=$lock && \ + echo lock_id=$lock_id && \ + if [ -n "$lock_id" ]; then + terraform force-unlock -force $lock_id + fi && \ + echo "Done"`, + } + + err := runUnlockTerraformDebugPod(h.clientset, clusterName, namespace, name, c, command) + if err != nil { + c.JSON(http.StatusUnprocessableEntity, response(http.StatusUnprocessableEntity, fmt.Sprintf("terraform unlock failed: %s", err), nil)) + return + } + + c.JSON(http.StatusOK, response(http.StatusOK, "terraform unlocked", nil)) + +} + type wsWrapper struct { *websocket.Conn } @@ -861,7 +920,7 @@ func (t TermSizer) Next() *remotecommand.TerminalSize { // } // command string, argv []string, headers map[string][]string, options ...Option -func New(clientset kubernetes.Interface, clusterName, namespace, name string, c *gin.Context, cmd []string) (*PodExec, error) { +func newSessionInTerraformDebugPod(clientset kubernetes.Interface, clusterName, namespace, name string, c *gin.Context, cmd, execCommand []string) (*PodExec, error) { pty, tty, err := ptylib.Open() if err != nil { log.Fatal(err) @@ -879,7 +938,7 @@ func New(clientset kubernetes.Interface, clusterName, namespace, name string, c go func() { defer pty.Close() - err := RemoteDebug(clientset, clusterName, namespace, name, tty, c, termSizer, cmd) + err := RemoteDebug(clientset, clusterName, namespace, name, tty, c, termSizer, cmd, execCommand) log.Println("Pod exec exited") closeCh <- err }() @@ -928,32 +987,59 @@ func (p *PodExec) Close() error { return nil } -// RemoteDebug starts the debug pod and connects in a tty that will be synced thru a websocket. Anything written to -// stdout will be synced to the tty. stderr logs will show up in the api logs and not the tty. -func RemoteDebug(parentClientset kubernetes.Interface, clusterName, namespace, name string, tty *os.File, c *gin.Context, terminalSizeQueue remotecommand.TerminalSizeQueue, cmd []string) error { +func createDebugPodManifest(c *gin.Context, config *rest.Config, namespace, name string, command []string) (*corev1.Pod, error) { + tfoclientset := tfo.NewForConfigOrDie(config) + tfclient := tfoclientset.TfV1beta1().Terraforms(namespace) + tf, err := tfclient.Get(c, name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + pod := generatePod(tf, command) + return pod, nil +} +func runUnlockTerraformDebugPod(parentClientset kubernetes.Interface, clusterName, namespace, name string, c *gin.Context, command []string) error { config, err := getVclusterConfig(parentClientset, "internal", clusterName) if err != nil { return err } - tfoclientset := tfo.NewForConfigOrDie(config) - clientset := kubernetes.NewForConfigOrDie(config) - tfclient := tfoclientset.TfV1beta1().Terraforms(namespace) - // newSession() - // tfoclientset := session.tfoclientset.TfV1beta1().Terraforms(namespace) + pod, err := createDebugPodManifest(c, config, namespace, name, command) + if err != nil { + return err + } + + clientset := kubernetes.NewForConfigOrDie(config) podClient := clientset.CoreV1().Pods(namespace) + pod, err = podClient.Create(c, pod, metav1.CreateOptions{}) + if err != nil { + return err + } - tf, err := tfclient.Get(c, name, metav1.GetOptions{}) + return getPodStatus(pod, clientset, namespace) +} + +// RemoteDebug starts the debug pod and connects in a tty that will be synced thru a websocket. Anything written to +// stdout will be synced to the tty. stderr logs will show up in the api logs and not the tty. +func RemoteDebug(parentClientset kubernetes.Interface, clusterName, namespace, name string, tty *os.File, c *gin.Context, terminalSizeQueue remotecommand.TerminalSizeQueue, cmd, execCommand []string) error { + + config, err := getVclusterConfig(parentClientset, "internal", clusterName) + if err != nil { + return err + } + + pod, err := createDebugPodManifest(c, config, namespace, name, nil) if err != nil { return err } - pod := generatePod(tf) + clientset := kubernetes.NewForConfigOrDie(config) + podClient := clientset.CoreV1().Pods(namespace) pod, err = podClient.Create(c, pod, metav1.CreateOptions{}) if err != nil { return err } + defer podClient.Delete(c, pod.Name, metav1.DeleteOptions{}) fmt.Printf("Connecting to %s ", pod.Name) @@ -1001,18 +1087,6 @@ func RemoteDebug(parentClientset kubernetes.Interface, clusterName, namespace, n } // log.Println(file.Name()) // log.Println("Setting up request") - execCommand := []string{ - "/bin/bash", - "-c", - `cd $TFO_MAIN_MODULE && \ - export PS1="\\w\\$ " && \ - if [[ -n "$AWS_WEB_IDENTITY_TOKEN_FILE" ]]; then - export $(irsa-tokengen); - echo printf "\nAWS creds set from token file\n" - fi && \ - printf "\nTry running 'terraform init'\n\n" && bash - `, - } if len(cmd) > 0 { execCommand = cmd @@ -1061,6 +1135,60 @@ func RemoteDebug(parentClientset kubernetes.Interface, clusterName, namespace, n return nil } +func getPodStatus(pod *corev1.Pod, clientset *kubernetes.Clientset, namespace string) error { + podStatusStartTime := time.Now() + for { + pod, err := clientset.CoreV1().Pods(namespace).Get(context.TODO(), pod.Name, metav1.GetOptions{}) + if err != nil { + return err + } + phase := pod.Status.Phase + if phase == "Succeeded" { + err := deletePod(namespace, pod.Name, clientset) + if err != nil { + return err + } + return nil + } else if phase == "Failed" { + err := deletePod(namespace, pod.Name, clientset) + if err != nil { + return err + } + return fmt.Errorf("Pod failed: phase %s", phase) + } + time.Sleep(5 * time.Second) + isDeleted, err := podTimeToLive(podStatusStartTime, pod.Name, namespace, clientset, 300) + if err != nil { + return err + } + if isDeleted == true { + return fmt.Errorf("Pod did not complete in time and was forcefully deleted") + } + + } + +} + +func deletePod(namespace, podName string, clientset *kubernetes.Clientset) error { + err := clientset.CoreV1().Pods(namespace).Delete(context.TODO(), podName, metav1.DeleteOptions{}) + if err != nil { + return err + } + return nil +} + +func podTimeToLive(podStatusStartTime time.Time, podName, namespace string, clientset *kubernetes.Clientset, timeToLive time.Duration) (bool, error) { + var deleteCompleted = false + if time.Since(podStatusStartTime) >= timeToLive*time.Second { + err := deletePod(namespace, podName, clientset) + if err != nil { + return deleteCompleted, err + } + deleteCompleted = true + } + return deleteCompleted, nil +} + // func isTerminal(file *os.File) bool { // inFd := file.Fd() @@ -1070,7 +1198,7 @@ func RemoteDebug(parentClientset kubernetes.Interface, clusterName, namespace, n // } -func generatePod(tf *tfv1beta1.Terraform) *corev1.Pod { +func generatePod(tf *tfv1beta1.Terraform, command []string) *corev1.Pod { terraformVersion := tf.Spec.TerraformVersion if terraformVersion == "" { terraformVersion = "1.1.5" @@ -1230,13 +1358,17 @@ func generatePod(tf *tfv1beta1.Terraform) *corev1.Pod { } restartPolicy := corev1.RestartPolicyNever + if command == nil { + command = []string{ + "/bin/sleep", "86400", + } + } + containers = append(containers, corev1.Container{ SecurityContext: securityContext, Name: "debug", Image: "ghcr.io/galleybytes/terraform-operator-tftaskv1.1.0:" + terraformVersion, - Command: []string{ - "/bin/sleep", "86400", - }, + Command: command, ImagePullPolicy: corev1.PullIfNotPresent, EnvFrom: envFrom, Env: env,