From 38c9f19886251b5bbdd06a6391db1c146d90ce29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paolo=20Chil=C3=A0?= Date: Mon, 24 Jan 2022 17:14:05 +0100 Subject: [PATCH] fix: clean-up pods when deleting kubernetes jobs (#154) (#155) Signed-off-by: Paolo Chila --- README.md | 7 +++++++ pkg/eventhandler/eventhandlers.go | 15 +++++---------- pkg/k8sutils/connect.go | 7 ++++--- pkg/k8sutils/fake/connect_mock.go | 5 ++++- pkg/k8sutils/job.go | 14 +++++--------- 5 files changed, 25 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index e1a0c439..cc77f78b 100644 --- a/README.md +++ b/README.md @@ -691,6 +691,13 @@ tasks: ttlSecondsAfterFinished: 600 ``` +**Note:** `ttlSecondsAfterFinished` relies on setting the [same property](https://kubernetes.io/docs/concepts/workloads/controllers/job/#ttl-mechanism-for-finished-jobs) +in kubernetes job workloads spec. The TTL controller (alpha from Kubernetes v1.12-v1.20, beta in v1.21-v1.22, GA in v1.23) +will then take care of cleanup. +More information about feature stages in kubernetes (that is what alpha, beta and GA implies in that context) have a look at +the [official kubernetes documentation](https://v1-20.docs.kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-stages). + + ## How to validate a job configuration `job-lint` is a simple cli tool that validates any given job configuration file and shows possible errors. diff --git a/pkg/eventhandler/eventhandlers.go b/pkg/eventhandler/eventhandlers.go index 85835a22..20520e87 100644 --- a/pkg/eventhandler/eventhandlers.go +++ b/pkg/eventhandler/eventhandlers.go @@ -3,13 +3,14 @@ package eventhandler import ( "encoding/json" "fmt" - "keptn-contrib/job-executor-service/pkg/config" - "keptn-contrib/job-executor-service/pkg/k8sutils" "log" "math" "strconv" "time" + "keptn-contrib/job-executor-service/pkg/config" + "keptn-contrib/job-executor-service/pkg/k8sutils" + cloudevents "github.com/cloudevents/sdk-go/v2" // make sure to use v2 cloudevents here keptnv2 "github.com/keptn/go-utils/pkg/lib/v0_2_0" ) @@ -146,7 +147,8 @@ func (eh *EventHandler) startK8sJob(k8s k8sutils.K8s, action *config.Action, jso namespace = task.Namespace } - err := k8s.CreateK8sJob(jobName, action, task, eh.EventData, eh.JobSettings, jsonEventData, namespace) + err := k8s.CreateK8sJob(jobName, action, task, eh.EventData, eh.JobSettings, + jsonEventData, namespace) if err != nil { log.Printf("Error while creating job: %s\n", err) @@ -156,13 +158,6 @@ func (eh *EventHandler) startK8sJob(k8s k8sutils.K8s, action *config.Action, jso return } - defer func() { - err = k8s.DeleteK8sJob(jobName, namespace) - if err != nil { - log.Printf("Error while deleting job: %s\n", err.Error()) - } - }() - maxPollCount := defaultMaxPollCount if task.MaxPollDuration != nil { maxPollCount = int(math.Ceil(float64(*task.MaxPollDuration) / pollIntervalInSeconds)) diff --git a/pkg/k8sutils/connect.go b/pkg/k8sutils/connect.go index 22fa5466..89455491 100644 --- a/pkg/k8sutils/connect.go +++ b/pkg/k8sutils/connect.go @@ -18,10 +18,11 @@ type k8sImpl struct { // K8s is used to interact with kubernetes jobs type K8s interface { ConnectToCluster() error - CreateK8sJob(jobName string, action *config.Action, task config.Task, eventData *keptnv2.EventData, - jobSettings JobSettings, jsonEventData interface{}, namespace string) error + CreateK8sJob( + jobName string, action *config.Action, task config.Task, eventData *keptnv2.EventData, jobSettings JobSettings, + jsonEventData interface{}, namespace string, + ) error AwaitK8sJobDone(jobName string, maxPollDuration int, pollIntervalInSeconds int, namespace string) error - DeleteK8sJob(jobName string, namespace string) error GetLogsOfPod(jobName string, namespace string) (string, error) } diff --git a/pkg/k8sutils/fake/connect_mock.go b/pkg/k8sutils/fake/connect_mock.go index a4470ee9..a877a300 100644 --- a/pkg/k8sutils/fake/connect_mock.go +++ b/pkg/k8sutils/fake/connect_mock.go @@ -65,7 +65,10 @@ func (mr *MockK8sMockRecorder) ConnectToCluster() *gomock.Call { } // CreateK8sJob mocks base method. -func (m *MockK8s) CreateK8sJob(jobName string, action *config.Action, task config.Task, eventData *v0_2_0.EventData, jobSettings k8sutils.JobSettings, jsonEventData interface{}, namespace string) error { +func (m *MockK8s) CreateK8sJob( + jobName string, action *config.Action, task config.Task, eventData *v0_2_0.EventData, + jobSettings k8sutils.JobSettings, jsonEventData interface{}, namespace string, +) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CreateK8sJob", jobName, action, task, eventData, jobSettings, jsonEventData, namespace) ret0, _ := ret[0].(error) diff --git a/pkg/k8sutils/job.go b/pkg/k8sutils/job.go index 7855f146..7e078a2e 100644 --- a/pkg/k8sutils/job.go +++ b/pkg/k8sutils/job.go @@ -38,8 +38,11 @@ type JobSettings struct { } // CreateK8sJob creates a k8s job with the job-executor-service-initcontainer and the job image of the task -func (k8s *k8sImpl) CreateK8sJob(jobName string, action *config.Action, task config.Task, eventData *keptnv2.EventData, - jobSettings JobSettings, jsonEventData interface{}, namespace string) error { +// returns ttlSecondsAfterFinished as stored in k8s or error in case of issues creating the job +func (k8s *k8sImpl) CreateK8sJob( + jobName string, action *config.Action, task config.Task, eventData *keptnv2.EventData, jobSettings JobSettings, + jsonEventData interface{}, namespace string, +) error { var backOffLimit int32 = 0 @@ -240,13 +243,6 @@ func (k8s *k8sImpl) AwaitK8sJobDone(jobName string, maxPollCount int, pollInterv } } -// DeleteK8sJob delete a k8s job in the given namespace -func (k8s *k8sImpl) DeleteK8sJob(jobName string, namespace string) error { - - jobs := k8s.clientset.BatchV1().Jobs(namespace) - return jobs.Delete(context.TODO(), jobName, metav1.DeleteOptions{}) -} - func (k8s *k8sImpl) prepareJobEnv(task config.Task, eventData *keptnv2.EventData, jsonEventData interface{}, namespace string) ([]v1.EnvVar, error) { var jobEnv []v1.EnvVar