fix(harness): retry CompleteStackRun with less data on entity too large

internal/errors/base.go

@@ -36,6 +36,14 @@ func (er *wrappedErrorResponse) Has(err KnownError) bool {
 	return false
 }
 
+func (er *wrappedErrorResponse) HasNetworkError(code int) bool {
+	if er.err.NetworkError == nil {
+		return false
+	}
+
+	return er.err.NetworkError.Code == code
+}
+
 func newAPIError(err *client.ErrorResponse) *wrappedErrorResponse {
 	return &wrappedErrorResponse{
 		err: err,
@@ -69,3 +77,17 @@ func IsDeleteRepository(err error) bool {
 
 	return newAPIError(errorResponse).Has(ErrDeleteRepository)
 }
+
+func IsNetworkError(err error, code int) bool {
+	if err == nil {
+		return false
+	}
+
+	errorResponse := new(client.ErrorResponse)
+	ok := errors.As(err, &errorResponse)
+	if !ok {
+		return false
+	}
+
+	return newAPIError(errorResponse).HasNetworkError(code)
+}

pkg/harness/controller/controller.go

@@ -16,6 +16,7 @@ import (
 	"github.com/pluralsh/deployment-operator/pkg/harness/environment"
 	"github.com/pluralsh/deployment-operator/pkg/harness/exec"
 	"github.com/pluralsh/deployment-operator/pkg/harness/sink"
+	"github.com/pluralsh/deployment-operator/pkg/harness/stackrun"
 	v1 "github.com/pluralsh/deployment-operator/pkg/harness/stackrun/v1"
 	"github.com/pluralsh/deployment-operator/pkg/harness/tool"
 	"github.com/pluralsh/deployment-operator/pkg/log"
@@ -139,14 +140,6 @@ func (in *stackRunController) toExecutable(ctx context.Context, step *gqlclient.
 		exec.WithOutputAnalyzer(exec.NewKeywordDetector()),
 	)
 
-	// Add a custom run step output analyzer for the destroy stage to increase
-	// a chance of detecting errors during execution. On occasion executable can
-	// return exit code 0 even though there was a fatal error during execution.
-	// TODO: use destroy stage
-	// if step.Stage == gqlclient.StepStageApply {
-	//	options = append(options, exec.WithOutputAnalyzer(exec.NewKeywordDetector()))
-	//}
-
 	return exec.NewExecutable(step.Cmd, options...)
 }
 
@@ -179,7 +172,7 @@ func (in *stackRunController) completeStackRun(status gqlclient.StackStatus, sta
 		})
 	}
 
-	return in.consoleClient.CompleteStackRun(in.stackRunID, gqlclient.StackRunAttributes{
+	return stackrun.CompleteStackRun(in.consoleClient, in.stackRunID, &gqlclient.StackRunAttributes{
 		Errors: serviceErrorAttributes,
 		Output: output,
 		State:  state,

pkg/harness/stackrun/helpers.go

@@ -2,12 +2,15 @@ package stackrun
 
 import (
 	"context"
+	"fmt"
+	"net/http"
 	"time"
 
 	gqlclient "github.com/pluralsh/console-client-go"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/klog/v2"
 
+	"github.com/pluralsh/deployment-operator/internal/errors"
 	console "github.com/pluralsh/deployment-operator/pkg/client"
 )
 
@@ -34,8 +37,63 @@ func StartStackRun(client console.Client, id string) error {
 	return MarkStackRun(client, id, gqlclient.StackStatusRunning)
 }
 
-func CompleteStackRun(client console.Client, id string) error {
-	return MarkStackRun(client, id, gqlclient.StackStatusSuccessful)
+func CompleteStackRun(client console.Client, id string, attributes *gqlclient.StackRunAttributes) (err error) {
+	if attributes == nil {
+		return fmt.Errorf("cannot complete stack run with nil attributes")
+	}
+
+	createModifierIter := func() func() func(attributes *gqlclient.StackRunAttributes) *gqlclient.StackRunAttributes {
+		idx := 0
+		modifiers := []func(attributes *gqlclient.StackRunAttributes) *gqlclient.StackRunAttributes{
+			func(attributes *gqlclient.StackRunAttributes) *gqlclient.StackRunAttributes {
+				if attributes.State == nil {
+					return attributes
+				}
+
+				// first drop the state
+				attributes.State.State = nil
+				return attributes
+			},
+			func(attributes *gqlclient.StackRunAttributes) *gqlclient.StackRunAttributes {
+				if attributes.State == nil {
+					return attributes
+				}
+
+				// next drop the plan
+				attributes.State.Plan = nil
+				return attributes
+			},
+			func(attributes *gqlclient.StackRunAttributes) *gqlclient.StackRunAttributes {
+				// as the last resort drop output
+				attributes.Output = nil
+				return attributes
+			},
+		}
+
+		return func() func(attributes *gqlclient.StackRunAttributes) *gqlclient.StackRunAttributes {
+			if idx > len(modifiers) {
+				return nil
+			}
+
+			m := modifiers[idx]
+			idx++
+			return m
+		}
+	}
+
+	next := createModifierIter()
+	for {
+		if err = client.CompleteStackRun(id, *attributes); !errors.IsNetworkError(err, http.StatusRequestEntityTooLarge) {
+			return err
+		}
+
+		if modifier := next(); modifier != nil {
+			attributes = modifier(attributes)
+			continue
+		}
+
+		return err
+	}
 }
 
 func CancelStackRun(client console.Client, id string) error {