Succeed RC deletion when resource not found (#123)

sky-uk · Jun 20, 2022 · d0fae7b · d0fae7b
1 parent d9426a1
commit d0fae7b
Show file tree

Hide file tree

Showing 2 changed files with 26 additions and 5 deletions.
diff --git a/controllers/pipelines/runconfiguration_workflow_factory.go b/controllers/pipelines/runconfiguration_workflow_factory.go
@@ -219,9 +219,11 @@ func (workflows *RunConfigurationWorkflowFactory) deleter(runConfiguration *pipe
 		return argo.Template{}, err
 	}
 
+	succeedOnNotFound := fmt.Sprintf(`KFP_RESULT=$(%s 2>&1) || echo $KFP_RESULT | grep -o 'HTTP response body: {.*}' | cut -d ':' -f 2- | jq -e 'select(.code==5)'`, kfpScript)
+
 	return argo.Template{
 		Name:     RunConfigurationWorkflowConstants.DeletionStepName,
 		Metadata: workflows.Config.Argo.MetadataDefaults,
-		Script:   workflows.ScriptTemplate(kfpScript),
+		Script:   workflows.ScriptTemplate(succeedOnNotFound),
 	}, nil
 }
diff --git a/controllers/pipelines/runconfiguration_workflow_integration_test.go b/controllers/pipelines/runconfiguration_workflow_integration_test.go
@@ -97,10 +97,10 @@ var _ = Context("RunConfiguration Workflows", Serial, func() {
 			))
 	}
 
-	var FailDeletion = func(jobKfpId string) error {
+	var FailDeletionWithCode = func(jobKfpId string, code int64) error {
 		return wiremockClient.StubFor(wiremock.Delete(wiremock.URLPathEqualTo("/apis/v1beta1/jobs/"+jobKfpId)).
 			WillReturn(
-				`{"status": "failed"}`,
+				fmt.Sprintf(`HTTP response body: {"status": "failed", "code": %d}`, code),
 				map[string]string{"Content-Type": "application/json"},
 				404,
 			))
@@ -174,13 +174,22 @@ var _ = Context("RunConfiguration Workflows", Serial, func() {
 		),
 		Entry("Deletion fails",
 			func(runconfiguration *pipelinesv1.RunConfiguration) {
-				Expect(FailDeletion(jobKfpId)).To(Succeed())
+				Expect(FailDeletionWithCode(jobKfpId, 7)).To(Succeed())
 			},
 			workflowFactory.ConstructDeletionWorkflow,
 			func(g Gomega, workflow *argo.Workflow) {
 				g.Expect(workflow.Status.Phase).To(Equal(argo.WorkflowFailed))
 			},
 		),
+		Entry("Deletion fails with RC not found",
+			func(runconfiguration *pipelinesv1.RunConfiguration) {
+				Expect(FailDeletionWithCode(jobKfpId, 5)).To(Succeed())
+			},
+			workflowFactory.ConstructDeletionWorkflow,
+			func(g Gomega, workflow *argo.Workflow) {
+				g.Expect(workflow.Status.Phase).To(Equal(argo.WorkflowSucceeded))
+			},
+		),
 	)
 
 	DescribeTable("Update Workflow", AssertWorkflow,
@@ -205,11 +214,21 @@ var _ = Context("RunConfiguration Workflows", Serial, func() {
 					To(Equal(""))
 			}),
 		Entry("Deletion fails", func(runconfiguration *pipelinesv1.RunConfiguration) {
-			Expect(FailDeletion(jobKfpId)).To(Succeed())
+			Expect(FailDeletionWithCode(jobKfpId, 55)).To(Succeed())
 		},
 			workflowFactory.ConstructUpdateWorkflow,
 			func(g Gomega, workflow *argo.Workflow) {
 				g.Expect(workflow.Status.Phase).To(Equal(argo.WorkflowFailed))
 			}),
+		Entry("Deletion fails with RC not found and creation succeeds", func(runconfiguration *pipelinesv1.RunConfiguration) {
+			Expect(FailDeletionWithCode(jobKfpId, 5)).To(Succeed())
+			Expect(SucceedCreation(runconfiguration, newJobKfpId)).To(Succeed())
+		},
+			workflowFactory.ConstructUpdateWorkflow,
+			func(g Gomega, workflow *argo.Workflow) {
+				g.Expect(workflow.Status.Phase).To(Equal(argo.WorkflowSucceeded))
+				g.Expect(getWorkflowOutput(workflow, RunConfigurationWorkflowConstants.RunConfigurationIdParameterName)).
+					To(Equal(newJobKfpId))
+			}),
 	)
 })