From dc096ac9208d3d718edfaad407433007b58bdad1 Mon Sep 17 00:00:00 2001 From: Mike Rostermund Date: Mon, 29 Jan 2024 16:53:15 +0100 Subject: [PATCH 1/3] Bump humio/cli dependency to fix bug with repository deletion --- .../resources/humioresources_controller_test.go | 4 ++++ controllers/suite/resources/suite_test.go | 1 + go.mod | 4 ++-- go.sum | 13 ++++--------- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/controllers/suite/resources/humioresources_controller_test.go b/controllers/suite/resources/humioresources_controller_test.go index 7340a227..e0014652 100644 --- a/controllers/suite/resources/humioresources_controller_test.go +++ b/controllers/suite/resources/humioresources_controller_test.go @@ -328,6 +328,7 @@ var _ = Describe("Humio Resources Controllers", func() { IngestSizeInGB: 5, StorageSizeInGB: 1, }, + AllowDataDeletion: true, }, } @@ -434,6 +435,7 @@ var _ = Describe("Humio Resources Controllers", func() { IngestSizeInGB: 5, StorageSizeInGB: 1, }, + AllowDataDeletion: true, }, } @@ -779,6 +781,7 @@ var _ = Describe("Humio Resources Controllers", func() { Spec: humiov1alpha1.HumioRepositorySpec{ ManagedClusterName: "non-existent-managed-cluster", Name: "parsername", + AllowDataDeletion: true, }, } Expect(k8sClient.Create(ctx, toCreateRepository)).Should(Succeed()) @@ -812,6 +815,7 @@ var _ = Describe("Humio Resources Controllers", func() { Spec: humiov1alpha1.HumioRepositorySpec{ ExternalClusterName: "non-existent-external-cluster", Name: "parsername", + AllowDataDeletion: true, }, } Expect(k8sClient.Create(ctx, toCreateRepository)).Should(Succeed()) diff --git a/controllers/suite/resources/suite_test.go b/controllers/suite/resources/suite_test.go index b8464cf0..e5ec1f1a 100644 --- a/controllers/suite/resources/suite_test.go +++ b/controllers/suite/resources/suite_test.go @@ -248,6 +248,7 @@ var _ = BeforeSuite(func() { Spec: corev1alpha1.HumioRepositorySpec{ ManagedClusterName: clusterKey.Name, Name: "test-repo", + AllowDataDeletion: true, }, } Expect(k8sClient.Create(context.TODO(), &testRepo)).To(Succeed()) diff --git a/go.mod b/go.mod index 9db596d3..251b8b5b 100644 --- a/go.mod +++ b/go.mod @@ -5,12 +5,12 @@ go 1.20 require ( github.com/Masterminds/semver v1.5.0 github.com/cert-manager/cert-manager v1.12.4 - github.com/cli/shurcooL-graphql v0.0.3 + github.com/cli/shurcooL-graphql v0.0.4 github.com/go-logr/logr v1.2.4 github.com/go-logr/zapr v1.2.4 github.com/google/go-cmp v0.5.9 github.com/google/martian v2.1.0+incompatible - github.com/humio/cli v0.32.4-0.20231025112913-b4dceeab38b6 + github.com/humio/cli v0.32.4-0.20240129154843-51063a956d0b github.com/onsi/ginkgo/v2 v2.12.0 github.com/onsi/gomega v1.27.11-0.20230807134635-babe25fc5472 github.com/prometheus/client_golang v1.16.0 diff --git a/go.sum b/go.sum index f2b3a6a9..d67029ad 100644 --- a/go.sum +++ b/go.sum @@ -11,8 +11,8 @@ github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cli/shurcooL-graphql v0.0.3 h1:CtpPxyGDs136/+ZeyAfUKYmcQBjDlq5aqnrDCW5Ghh8= -github.com/cli/shurcooL-graphql v0.0.3/go.mod h1:tlrLmw/n5Q/+4qSvosT+9/W5zc8ZMjnJeYBxSdb4nWA= +github.com/cli/shurcooL-graphql v0.0.4 h1:6MogPnQJLjKkaXPyGqPRXOI2qCsQdqNfUY1QSJu2GuY= +github.com/cli/shurcooL-graphql v0.0.4/go.mod h1:3waN4u02FiZivIV+p1y4d0Jo1jc6BViMA73C+sZo2fk= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -62,8 +62,8 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJY github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/humio/cli v0.32.4-0.20231025112913-b4dceeab38b6 h1:fEb+4t483D2QPA4CtcGxCV3SJ4IpGEN3OaU0i7rBMr0= -github.com/humio/cli v0.32.4-0.20231025112913-b4dceeab38b6/go.mod h1:I4yilQO5wI6uc7NMLmKSGFATY3AZddCbehIvjESK8WQ= +github.com/humio/cli v0.32.4-0.20240129154843-51063a956d0b h1:mlyn4bxK8cbbtkDeBu0oIHdrl9UkX8j/+2Souify/pw= +github.com/humio/cli v0.32.4-0.20240129154843-51063a956d0b/go.mod h1:T4mOVMdyJcHM5ANBx8U5/7cbqV5K+O7hWQd9Q78nd7U= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= @@ -151,7 +151,6 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20220923203811-8be639271d50/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= @@ -170,19 +169,15 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= From 41389ebde23a57e681c22b5900bc11ff18e10210 Mon Sep 17 00:00:00 2001 From: Mike Rostermund Date: Mon, 29 Jan 2024 16:54:18 +0100 Subject: [PATCH 2/3] Add k8s object UID to log entries Intention is to ensure we have an easy way to understand if objects keeps getting deleted and recreated over and over. --- controllers/humioaction_controller.go | 2 ++ controllers/humioalert_controller.go | 2 ++ controllers/humiocluster_controller.go | 2 ++ controllers/humioexternalcluster_controller.go | 2 ++ controllers/humioingesttoken_controller.go | 2 ++ controllers/humioparser_controller.go | 2 ++ controllers/humiorepository_controller.go | 2 ++ controllers/humioview_controller.go | 2 ++ 8 files changed, 16 insertions(+) diff --git a/controllers/humioaction_controller.go b/controllers/humioaction_controller.go index 77655878..ae6c43d4 100644 --- a/controllers/humioaction_controller.go +++ b/controllers/humioaction_controller.go @@ -70,6 +70,8 @@ func (r *HumioActionReconciler) Reconcile(ctx context.Context, req ctrl.Request) return reconcile.Result{}, err } + r.Log = r.Log.WithValues("Request.UID", ha.UID) + cluster, err := helpers.NewCluster(ctx, r, ha.Spec.ManagedClusterName, ha.Spec.ExternalClusterName, ha.Namespace, helpers.UseCertManager(), true) if err != nil || cluster == nil || cluster.Config() == nil { r.Log.Error(err, "unable to obtain humio client config") diff --git a/controllers/humioalert_controller.go b/controllers/humioalert_controller.go index cfaf559d..83db7885 100644 --- a/controllers/humioalert_controller.go +++ b/controllers/humioalert_controller.go @@ -73,6 +73,8 @@ func (r *HumioAlertReconciler) Reconcile(ctx context.Context, req ctrl.Request) return reconcile.Result{}, err } + r.Log = r.Log.WithValues("Request.UID", ha.UID) + cluster, err := helpers.NewCluster(ctx, r, ha.Spec.ManagedClusterName, ha.Spec.ExternalClusterName, ha.Namespace, helpers.UseCertManager(), true) if err != nil || cluster == nil || cluster.Config() == nil { r.Log.Error(err, "unable to obtain humio client config") diff --git a/controllers/humiocluster_controller.go b/controllers/humiocluster_controller.go index bdaf5d79..6a940409 100644 --- a/controllers/humiocluster_controller.go +++ b/controllers/humiocluster_controller.go @@ -98,6 +98,8 @@ func (r *HumioClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request return reconcile.Result{}, err } + r.Log = r.Log.WithValues("Request.UID", hc.UID) + var humioNodePools HumioNodePoolList humioNodePools.Add(NewHumioNodeManagerFromHumioCluster(hc)) for idx := range hc.Spec.NodePools { diff --git a/controllers/humioexternalcluster_controller.go b/controllers/humioexternalcluster_controller.go index 5b28d064..7ac083f5 100644 --- a/controllers/humioexternalcluster_controller.go +++ b/controllers/humioexternalcluster_controller.go @@ -70,6 +70,8 @@ func (r *HumioExternalClusterReconciler) Reconcile(ctx context.Context, req ctrl return reconcile.Result{}, err } + r.Log = r.Log.WithValues("Request.UID", hec.UID) + if hec.Status.State == "" { err := r.setState(ctx, humiov1alpha1.HumioExternalClusterStateUnknown, hec) if err != nil { diff --git a/controllers/humioingesttoken_controller.go b/controllers/humioingesttoken_controller.go index 40a57988..2fec34d7 100644 --- a/controllers/humioingesttoken_controller.go +++ b/controllers/humioingesttoken_controller.go @@ -74,6 +74,8 @@ func (r *HumioIngestTokenReconciler) Reconcile(ctx context.Context, req ctrl.Req return reconcile.Result{}, err } + r.Log = r.Log.WithValues("Request.UID", hit.UID) + cluster, err := helpers.NewCluster(ctx, r, hit.Spec.ManagedClusterName, hit.Spec.ExternalClusterName, hit.Namespace, helpers.UseCertManager(), true) if err != nil || cluster == nil || cluster.Config() == nil { r.Log.Error(err, "unable to obtain humio client config") diff --git a/controllers/humioparser_controller.go b/controllers/humioparser_controller.go index b049d2dd..f7ad2205 100644 --- a/controllers/humioparser_controller.go +++ b/controllers/humioparser_controller.go @@ -75,6 +75,8 @@ func (r *HumioParserReconciler) Reconcile(ctx context.Context, req ctrl.Request) return reconcile.Result{}, err } + r.Log = r.Log.WithValues("Request.UID", hp.UID) + cluster, err := helpers.NewCluster(ctx, r, hp.Spec.ManagedClusterName, hp.Spec.ExternalClusterName, hp.Namespace, helpers.UseCertManager(), true) if err != nil || cluster == nil || cluster.Config() == nil { r.Log.Error(err, "unable to obtain humio client config") diff --git a/controllers/humiorepository_controller.go b/controllers/humiorepository_controller.go index 4441769f..63575774 100644 --- a/controllers/humiorepository_controller.go +++ b/controllers/humiorepository_controller.go @@ -73,6 +73,8 @@ func (r *HumioRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Requ return reconcile.Result{}, err } + r.Log = r.Log.WithValues("Request.UID", hr.UID) + cluster, err := helpers.NewCluster(ctx, r, hr.Spec.ManagedClusterName, hr.Spec.ExternalClusterName, hr.Namespace, helpers.UseCertManager(), true) if err != nil || cluster == nil || cluster.Config() == nil { r.Log.Error(err, "unable to obtain humio client config") diff --git a/controllers/humioview_controller.go b/controllers/humioview_controller.go index d31c592c..8df776c0 100644 --- a/controllers/humioview_controller.go +++ b/controllers/humioview_controller.go @@ -73,6 +73,8 @@ func (r *HumioViewReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return reconcile.Result{}, err } + r.Log = r.Log.WithValues("Request.UID", hv.UID) + cluster, err := helpers.NewCluster(ctx, r, hv.Spec.ManagedClusterName, hv.Spec.ExternalClusterName, hv.Namespace, helpers.UseCertManager(), true) if err != nil || cluster == nil || cluster.Config() == nil { r.Log.Error(err, "unable to obtain humio client config") From 12c346621142b80c9f9496fcf7fdc5749c63bb2e Mon Sep 17 00:00:00 2001 From: Mike Rostermund Date: Mon, 29 Jan 2024 16:55:05 +0100 Subject: [PATCH 3/3] Only ignore IsNotFound k8s errors, and return any other error we may get --- controllers/humioingesttoken_controller.go | 7 +++++-- controllers/humioparser_controller.go | 7 +++++-- controllers/humiorepository_controller.go | 7 +++++-- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/controllers/humioingesttoken_controller.go b/controllers/humioingesttoken_controller.go index 2fec34d7..427c5892 100644 --- a/controllers/humioingesttoken_controller.go +++ b/controllers/humioingesttoken_controller.go @@ -190,8 +190,11 @@ func (r *HumioIngestTokenReconciler) SetupWithManager(mgr ctrl.Manager) error { func (r *HumioIngestTokenReconciler) finalize(ctx context.Context, config *humioapi.Config, req reconcile.Request, hit *humiov1alpha1.HumioIngestToken) error { _, err := helpers.NewCluster(ctx, r, hit.Spec.ManagedClusterName, hit.Spec.ExternalClusterName, hit.Namespace, helpers.UseCertManager(), true) - if k8serrors.IsNotFound(err) { - return nil + if err != nil { + if k8serrors.IsNotFound(err) { + return nil + } + return err } return r.HumioClient.DeleteIngestToken(config, req, hit) diff --git a/controllers/humioparser_controller.go b/controllers/humioparser_controller.go index f7ad2205..10aeae1e 100644 --- a/controllers/humioparser_controller.go +++ b/controllers/humioparser_controller.go @@ -195,8 +195,11 @@ func (r *HumioParserReconciler) SetupWithManager(mgr ctrl.Manager) error { func (r *HumioParserReconciler) finalize(ctx context.Context, config *humioapi.Config, req reconcile.Request, hp *humiov1alpha1.HumioParser) error { _, err := helpers.NewCluster(ctx, r, hp.Spec.ManagedClusterName, hp.Spec.ExternalClusterName, hp.Namespace, helpers.UseCertManager(), true) - if k8serrors.IsNotFound(err) { - return nil + if err != nil { + if k8serrors.IsNotFound(err) { + return nil + } + return err } return r.HumioClient.DeleteParser(config, req, hp) diff --git a/controllers/humiorepository_controller.go b/controllers/humiorepository_controller.go index 63575774..83144cc5 100644 --- a/controllers/humiorepository_controller.go +++ b/controllers/humiorepository_controller.go @@ -191,8 +191,11 @@ func (r *HumioRepositoryReconciler) SetupWithManager(mgr ctrl.Manager) error { func (r *HumioRepositoryReconciler) finalize(ctx context.Context, config *humioapi.Config, req reconcile.Request, hr *humiov1alpha1.HumioRepository) error { _, err := helpers.NewCluster(ctx, r, hr.Spec.ManagedClusterName, hr.Spec.ExternalClusterName, hr.Namespace, helpers.UseCertManager(), true) - if k8serrors.IsNotFound(err) { - return nil + if err != nil { + if k8serrors.IsNotFound(err) { + return nil + } + return err } return r.HumioClient.DeleteRepository(config, req, hr)