From 7a529d40dcd041a1633e6fd894ef4dd23c566fa7 Mon Sep 17 00:00:00 2001 From: Gerrit Date: Tue, 18 Apr 2023 14:28:00 +0200 Subject: [PATCH] Integration test and improvements on recreate strategy. (#35) --- api/v2/defaults/defaults.go | 6 - api/v2/types_firewallset.go | 4 + controllers/deployment/reconcile.go | 46 +- controllers/deployment/recreate.go | 41 +- controllers/deployment/rolling.go | 20 +- controllers/firewall/delete.go | 13 +- controllers/firewall/reconcile.go | 9 +- controllers/firewall/status.go | 6 +- controllers/monitor/reconcile.go | 4 +- controllers/set/reconcile.go | 10 +- integration/integration_test.go | 1963 ++++++++++++++++++--------- integration/suite_test.go | 3 + 12 files changed, 1445 insertions(+), 680 deletions(-) diff --git a/api/v2/defaults/defaults.go b/api/v2/defaults/defaults.go index 5ceb09b..7c3d788 100644 --- a/api/v2/defaults/defaults.go +++ b/api/v2/defaults/defaults.go @@ -79,9 +79,6 @@ func (r *firewallSetDefaulter) Default(ctx context.Context, obj runtime.Object) r.log.Info("defaulting firewallset resource", "name", f.GetName(), "namespace", f.GetNamespace()) - if f.Spec.Replicas == 0 { - f.Spec.Replicas = 1 - } if f.Spec.Selector == nil { f.Spec.Selector = f.Spec.Template.Labels } @@ -99,9 +96,6 @@ func (r *firewallDeploymentDefaulter) Default(ctx context.Context, obj runtime.O r.log.Info("defaulting firewalldeployment resource", "name", f.GetName(), "namespace", f.GetNamespace()) - if f.Spec.Replicas == 0 { - f.Spec.Replicas = 1 - } if f.Spec.Strategy == "" { f.Spec.Strategy = v2.StrategyRollingUpdate } diff --git a/api/v2/types_firewallset.go b/api/v2/types_firewallset.go index b19a4a8..edc8409 100644 --- a/api/v2/types_firewallset.go +++ b/api/v2/types_firewallset.go @@ -30,6 +30,10 @@ func FirewallManagedByTag() string { return fmt.Sprintf("%s=%s", FirewallControllerManagedByAnnotation, FirewallControllerManager) } +func (f FirewallDistance) Pointer() *FirewallDistance { + return &f +} + // FirewallSet contains the spec template of a firewall resource similar to a Kubernetes ReplicaSet and takes care that the desired amount of firewall replicas is running. // // +kubebuilder:object:root=true diff --git a/controllers/deployment/reconcile.go b/controllers/deployment/reconcile.go index 2581c1a..52289fb 100644 --- a/controllers/deployment/reconcile.go +++ b/controllers/deployment/reconcile.go @@ -29,15 +29,15 @@ func (c *controller) Reconcile(r *controllers.Ctx[*v2.FirewallDeployment]) error return fmt.Errorf("unable to get owned sets: %w", err) } - current, err := controllers.MaxRevisionOf(ownedSets) + latestSet, err := controllers.MaxRevisionOf(ownedSets) if err != nil { return err } - if current == nil { + if latestSet == nil { r.Log.Info("no firewall set is present, creating a new one") - _, err := c.createFirewallSet(r, v2.FirewallShortestDistance, 0) + _, err := c.createFirewallSet(r, 0, nil) if err != nil { return err } @@ -47,9 +47,9 @@ func (c *controller) Reconcile(r *controllers.Ctx[*v2.FirewallDeployment]) error switch s := r.Target.Spec.Strategy; s { case v2.StrategyRecreate: - err = c.recreateStrategy(r, ownedSets, current) + err = c.recreateStrategy(r, ownedSets, latestSet) case v2.StrategyRollingUpdate: - err = c.rollingUpdateStrategy(r, ownedSets, current) + err = c.rollingUpdateStrategy(r, ownedSets, latestSet) default: err = fmt.Errorf("unknown deployment strategy: %s", s) } @@ -64,10 +64,10 @@ func (c *controller) Reconcile(r *controllers.Ctx[*v2.FirewallDeployment]) error } // we are done with the update, give the set the shortest distance if this is not already the case - if current.Status.ReadyReplicas == current.Spec.Replicas && current.Spec.Distance != v2.FirewallShortestDistance { - current.Spec.Distance = v2.FirewallShortestDistance + if latestSet.Status.ReadyReplicas == latestSet.Spec.Replicas && latestSet.Spec.Distance != v2.FirewallShortestDistance { + latestSet.Spec.Distance = v2.FirewallShortestDistance - err := c.c.GetSeedClient().Update(r.Ctx, current) + err := c.c.GetSeedClient().Update(r.Ctx, latestSet) if err != nil { return fmt.Errorf("unable to swap latest set distance to %d: %w", v2.FirewallShortestDistance, err) } @@ -78,16 +78,23 @@ func (c *controller) Reconcile(r *controllers.Ctx[*v2.FirewallDeployment]) error return nil } -func (c *controller) createNextFirewallSet(r *controllers.Ctx[*v2.FirewallDeployment], current *v2.FirewallSet, distance v2.FirewallDistance) (*v2.FirewallSet, error) { - revision, err := controllers.NextRevision(current) +func (c *controller) createNextFirewallSet(r *controllers.Ctx[*v2.FirewallDeployment], set *v2.FirewallSet, ows *setOverrides) (*v2.FirewallSet, error) { + revision, err := controllers.NextRevision(set) if err != nil { return nil, err } - return c.createFirewallSet(r, distance, revision) + return c.createFirewallSet(r, revision, ows) } -func (c *controller) createFirewallSet(r *controllers.Ctx[*v2.FirewallDeployment], distance v2.FirewallDistance, revision int) (*v2.FirewallSet, error) { +type setOverrides struct { + // override default distance (shortest distance) + distance *v2.FirewallDistance + // override default replicas (inherited from set spec) + replicas *int +} + +func (c *controller) createFirewallSet(r *controllers.Ctx[*v2.FirewallDeployment], revision int, ows *setOverrides) (*v2.FirewallSet, error) { if lastCreation, ok := c.lastSetCreation[r.Target.Name]; ok && time.Since(lastCreation) < c.c.GetSafetyBackoff() { // this is just for safety reasons to prevent mass-allocations r.Log.Info("backing off from firewall set creation as last creation is only seconds ago", "ago", time.Since(lastCreation).String()) @@ -99,7 +106,18 @@ func (c *controller) createFirewallSet(r *controllers.Ctx[*v2.FirewallDeployment return nil, err } - name := fmt.Sprintf("%s-%s", r.Target.Name, uuid.String()[:5]) + var ( + name = fmt.Sprintf("%s-%s", r.Target.Name, uuid.String()[:5]) + distance = v2.FirewallShortestDistance + replicas = r.Target.Spec.Replicas + ) + + if ows != nil && ows.distance != nil { + distance = *ows.distance + } + if ows != nil && ows.replicas != nil { + replicas = *ows.replicas + } set := &v2.FirewallSet{ ObjectMeta: metav1.ObjectMeta{ @@ -114,7 +132,7 @@ func (c *controller) createFirewallSet(r *controllers.Ctx[*v2.FirewallDeployment Labels: r.Target.Labels, }, Spec: v2.FirewallSetSpec{ - Replicas: r.Target.Spec.Replicas, + Replicas: replicas, Template: r.Target.Spec.Template, Distance: distance, }, diff --git a/controllers/deployment/recreate.go b/controllers/deployment/recreate.go index c03c743..819a43a 100644 --- a/controllers/deployment/recreate.go +++ b/controllers/deployment/recreate.go @@ -2,45 +2,58 @@ package deployment import ( "fmt" + "time" v2 "github.com/metal-stack/firewall-controller-manager/api/v2" "github.com/metal-stack/firewall-controller-manager/controllers" + "github.com/metal-stack/metal-lib/pkg/pointer" ) // recreateStrategy first deletes the existing firewall sets and then creates a new one -func (c *controller) recreateStrategy(r *controllers.Ctx[*v2.FirewallDeployment], ownedSets []*v2.FirewallSet, current *v2.FirewallSet) error { - newSetRequired, err := c.isNewSetRequired(r, current) +func (c *controller) recreateStrategy(r *controllers.Ctx[*v2.FirewallDeployment], ownedSets []*v2.FirewallSet, latestSet *v2.FirewallSet) error { + newSetRequired, err := c.isNewSetRequired(r, latestSet) if err != nil { return err } if newSetRequired { - r.Log.Info("significant changes detected in the spec, cleaning up old sets then create new firewall set") + r.Log.Info("significant changes detected in the spec, create new scaled down firewall set, then cleaning up old sets") - err = c.deleteFirewallSets(r, ownedSets...) + set, err := c.createNextFirewallSet(r, latestSet, &setOverrides{ + replicas: pointer.Pointer(0), + }) if err != nil { return err } - newSet, err := c.createNextFirewallSet(r, current, v2.FirewallShortestDistance) - if err != nil { - return err - } + c.recorder.Eventf(set, "Normal", "Recreate", "recreated firewallset old: %s new: %s", latestSet.Name, set.Name) - c.recorder.Eventf(newSet, "Normal", "Recreate", "recreated firewallset old: %s new: %s", current.Name, newSet.Name) + latestSet = set + } - return nil + err = c.deleteFirewallSets(r, controllers.Except(ownedSets, latestSet)...) + if err != nil { + return err } - err = c.syncFirewallSet(r, current) + err = c.syncFirewallSet(r, latestSet) if err != nil { return fmt.Errorf("unable to update firewall set: %w", err) } - if current.Status.ReadyReplicas == current.Spec.Replicas { - cond := v2.NewCondition(v2.FirewallDeplomentProgressing, v2.ConditionTrue, "NewFirewallSetAvailable", fmt.Sprintf("FirewallSet %q has successfully progressed.", current.Name)) - r.Target.Status.Conditions.Set(cond) + if latestSet.Status.ReadyReplicas != latestSet.Spec.Replicas { + r.Log.Info("set replicas are not yet ready") + + if time.Since(latestSet.CreationTimestamp.Time) > c.c.GetProgressDeadline() { + cond := v2.NewCondition(v2.FirewallDeplomentProgressing, v2.ConditionFalse, "ProgressDeadlineExceeded", fmt.Sprintf("FirewallSet %q has timed out progressing.", latestSet.Name)) + r.Target.Status.Conditions.Set(cond) + } + + return nil } + cond := v2.NewCondition(v2.FirewallDeplomentProgressing, v2.ConditionTrue, "NewFirewallSetAvailable", fmt.Sprintf("FirewallSet %q has successfully progressed.", latestSet.Name)) + r.Target.Status.Conditions.Set(cond) + return nil } diff --git a/controllers/deployment/rolling.go b/controllers/deployment/rolling.go index 49174c0..1875002 100644 --- a/controllers/deployment/rolling.go +++ b/controllers/deployment/rolling.go @@ -9,8 +9,8 @@ import ( ) // rollingUpdateStrategy first creates a new set and deletes the old one's when the new one becomes ready -func (c *controller) rollingUpdateStrategy(r *controllers.Ctx[*v2.FirewallDeployment], ownedSets []*v2.FirewallSet, current *v2.FirewallSet) error { - newSetRequired, err := c.isNewSetRequired(r, current) +func (c *controller) rollingUpdateStrategy(r *controllers.Ctx[*v2.FirewallDeployment], ownedSets []*v2.FirewallSet, latestSet *v2.FirewallSet) error { + newSetRequired, err := c.isNewSetRequired(r, latestSet) if err != nil { return err } @@ -18,7 +18,9 @@ func (c *controller) rollingUpdateStrategy(r *controllers.Ctx[*v2.FirewallDeploy if newSetRequired { r.Log.Info("significant changes detected in the spec, creating new firewall set", "distance", v2.FirewallRollingUpdateSetDistance) - newSet, err := c.createNextFirewallSet(r, current, v2.FirewallRollingUpdateSetDistance) + newSet, err := c.createNextFirewallSet(r, latestSet, &setOverrides{ + distance: v2.FirewallRollingUpdateSetDistance.Pointer(), + }) if err != nil { return err } @@ -30,28 +32,28 @@ func (c *controller) rollingUpdateStrategy(r *controllers.Ctx[*v2.FirewallDeploy return c.cleanupIntermediateSets(r, ownedSets) } - err = c.syncFirewallSet(r, current) + err = c.syncFirewallSet(r, latestSet) if err != nil { return fmt.Errorf("unable to update firewall set: %w", err) } - if current.Status.ReadyReplicas != current.Spec.Replicas { + if latestSet.Status.ReadyReplicas != latestSet.Spec.Replicas { r.Log.Info("set replicas are not yet ready") - if time.Since(current.CreationTimestamp.Time) > c.c.GetProgressDeadline() { - cond := v2.NewCondition(v2.FirewallDeplomentProgressing, v2.ConditionFalse, "ProgressDeadlineExceeded", fmt.Sprintf("FirewallSet %q has timed out progressing.", current.Name)) + if time.Since(latestSet.CreationTimestamp.Time) > c.c.GetProgressDeadline() { + cond := v2.NewCondition(v2.FirewallDeplomentProgressing, v2.ConditionFalse, "ProgressDeadlineExceeded", fmt.Sprintf("FirewallSet %q has timed out progressing.", latestSet.Name)) r.Target.Status.Conditions.Set(cond) } return c.cleanupIntermediateSets(r, ownedSets) } - cond := v2.NewCondition(v2.FirewallDeplomentProgressing, v2.ConditionTrue, "NewFirewallSetAvailable", fmt.Sprintf("FirewallSet %q has successfully progressed.", current.Name)) + cond := v2.NewCondition(v2.FirewallDeplomentProgressing, v2.ConditionTrue, "NewFirewallSetAvailable", fmt.Sprintf("FirewallSet %q has successfully progressed.", latestSet.Name)) r.Target.Status.Conditions.Set(cond) r.Log.Info("ensuring old sets are cleaned up") - oldSets := controllers.Except(ownedSets, current) + oldSets := controllers.Except(ownedSets, latestSet) return c.deleteFirewallSets(r, oldSets...) } diff --git a/controllers/firewall/delete.go b/controllers/firewall/delete.go index bbba348..eeac8a0 100644 --- a/controllers/firewall/delete.go +++ b/controllers/firewall/delete.go @@ -8,8 +8,8 @@ import ( v2 "github.com/metal-stack/firewall-controller-manager/api/v2" "github.com/metal-stack/firewall-controller-manager/controllers" "github.com/metal-stack/metal-go/api/client/machine" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" ) func (c *controller) Delete(r *controllers.Ctx[*v2.Firewall]) error { @@ -41,7 +41,9 @@ func (c *controller) Delete(r *controllers.Ctx[*v2.Firewall]) error { resp, err := c.c.GetMetal().Machine().FreeMachine(machine.NewFreeMachineParams().WithID(*f.ID).WithContext(r.Ctx), nil) if err != nil { - return fmt.Errorf("firewall delete error: %w", err) + r.Log.Error(err, "firewall deletion failed") + + return controllers.RequeueAfter(5*time.Second, "firewall deletion failed, retrying") } r.Log.Info("deleted firewall", "firewall-name", f.Name, "id", *resp.Payload.ID) @@ -62,8 +64,11 @@ func (c *controller) deleteFirewallMonitor(ctx context.Context, fw *v2.Firewall) err := c.c.GetShootClient().Delete(ctx, mon) if err != nil { - return client.IgnoreNotFound(err) + if apierrors.IsNotFound(err) { + return nil + } + return err } - return nil + return controllers.RequeueAfter(1*time.Second, "waiting for firewall monitor to be deleted, requeuing") } diff --git a/controllers/firewall/reconcile.go b/controllers/firewall/reconcile.go index 69e5571..e5c64c0 100644 --- a/controllers/firewall/reconcile.go +++ b/controllers/firewall/reconcile.go @@ -19,15 +19,16 @@ import ( func (c *controller) Reconcile(r *controllers.Ctx[*v2.Firewall]) error { var f *models.V1FirewallResponse defer func() { + if err := c.setStatus(r, f); err != nil { + r.Log.Error(err, "unable to set firewall status") + } + mon, err := c.ensureFirewallMonitor(r) if err != nil { r.Log.Error(err, "unable to deploy firewall monitor") - // not returning, we can still try to update the status } - if err := c.setStatus(r, f, mon); err != nil { - r.Log.Error(err, "unable to set firewall status") - } + SetFirewallStatusFromMonitor(r.Target, mon) }() fws, err := c.firewallCache.Get(r.Ctx, r.Target) diff --git a/controllers/firewall/status.go b/controllers/firewall/status.go index 72871bc..8f3b490 100644 --- a/controllers/firewall/status.go +++ b/controllers/firewall/status.go @@ -11,7 +11,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func (c *controller) setStatus(r *controllers.Ctx[*v2.Firewall], f *models.V1FirewallResponse, mon *v2.FirewallMonitor) error { +func (c *controller) setStatus(r *controllers.Ctx[*v2.Firewall], f *models.V1FirewallResponse) error { var errs []error err := setMachineStatus(r.Target, f) @@ -24,8 +24,6 @@ func (c *controller) setStatus(r *controllers.Ctx[*v2.Firewall], f *models.V1Fir errs = append(errs, err) } - SetFirewallStatus(r.Target, mon) - r.Target.Status.ShootAccess = c.c.GetShootAccess() return errors.Join(errs...) @@ -118,7 +116,7 @@ func (c *controller) setFirewallNetworks(r *controllers.Ctx[*v2.Firewall], f *mo return nil } -func SetFirewallStatus(fw *v2.Firewall, mon *v2.FirewallMonitor) { +func SetFirewallStatusFromMonitor(fw *v2.Firewall, mon *v2.FirewallMonitor) { if v2.IsAnnotationTrue(fw, v2.FirewallNoControllerConnectionAnnotation) { cond := v2.NewCondition(v2.FirewallControllerConnected, v2.ConditionTrue, "NotChecking", "Not checking controller connection due to firewall annotation.") fw.Status.Conditions.Set(cond) diff --git a/controllers/monitor/reconcile.go b/controllers/monitor/reconcile.go index 2a81f63..a60abab 100644 --- a/controllers/monitor/reconcile.go +++ b/controllers/monitor/reconcile.go @@ -23,7 +23,7 @@ func (c *controller) Reconcile(r *controllers.Ctx[*v2.FirewallMonitor]) error { fw, err := c.updateFirewallStatus(r) if err != nil { r.Log.Error(err, "unable to update firewall status") - return controllers.RequeueAfter(3*time.Second, "unable to update firewall status, retrying: %w") + return controllers.RequeueAfter(3*time.Second, "unable to update firewall status, retrying") } err = c.offerFirewallControllerMigrationSecret(r, fw) @@ -53,7 +53,7 @@ func (c *controller) updateFirewallStatus(r *controllers.Ctx[*v2.FirewallMonitor return nil, fmt.Errorf("associated firewall of monitor not found: %w", err) } - firewall.SetFirewallStatus(fw, r.Target) + firewall.SetFirewallStatusFromMonitor(fw, r.Target) err = c.c.GetSeedClient().Status().Update(r.Ctx, fw) if err != nil { diff --git a/controllers/set/reconcile.go b/controllers/set/reconcile.go index b9b2894..6591ea3 100644 --- a/controllers/set/reconcile.go +++ b/controllers/set/reconcile.go @@ -31,7 +31,13 @@ func (c *controller) Reconcile(r *controllers.Ctx[*v2.FirewallSet]) error { // - the least important at the end of the slice can be popped off for deletion on scale down v2.SortFirewallsByImportance(ownedFirewalls) - for i, fw := range ownedFirewalls { + for i, ownedFw := range ownedFirewalls { + fw := &v2.Firewall{} + err := c.c.GetSeedClient().Get(r.Ctx, client.ObjectKeyFromObject(ownedFw), fw) + if err != nil { + return fmt.Errorf("error fetching firewall: %w", err) + } + fw.Spec = r.Target.Spec.Template.Spec // stagger firewall replicas to achieve active/standby behavior @@ -62,7 +68,7 @@ func (c *controller) Reconcile(r *controllers.Ctx[*v2.FirewallSet]) error { fw.Distance = distance - err := c.c.GetSeedClient().Update(r.Ctx, fw, &client.UpdateOptions{}) + err = c.c.GetSeedClient().Update(r.Ctx, fw, &client.UpdateOptions{}) if err != nil { return fmt.Errorf("error updating firewall spec: %w", err) } diff --git a/integration/integration_test.go b/integration/integration_test.go index 7941200..66b108c 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -6,12 +6,14 @@ import ( v2 "github.com/metal-stack/firewall-controller-manager/api/v2" "github.com/metal-stack/firewall-controller-manager/api/v2/defaults" + "github.com/metal-stack/metal-lib/httperrors" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/stretchr/testify/mock" "sigs.k8s.io/controller-runtime/pkg/client" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" testcommon "github.com/metal-stack/firewall-controller-manager/integration/common" @@ -24,7 +26,7 @@ import ( ) var ( - interval = 250 * time.Millisecond + interval = 200 * time.Millisecond namespace = &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ @@ -118,32 +120,35 @@ users: var _ = Context("integration test", Ordered, func() { var ( - deployment = &v2.FirewallDeployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - Namespace: namespaceName, - }, - Spec: v2.FirewallDeploymentSpec{ - Template: v2.FirewallTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "purpose": "shoot-firewall", + deployment = func() *v2.FirewallDeployment { + return &v2.FirewallDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: namespaceName, + }, + Spec: v2.FirewallDeploymentSpec{ + Replicas: 1, + Template: v2.FirewallTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "purpose": "shoot-firewall", + }, + }, + Spec: v2.FirewallSpec{ + Size: "n1-medium-x86", + Project: "project-a", + Partition: "partition-a", + Image: "firewall-ubuntu-2.0", + Networks: []string{"internet"}, + ControllerURL: "http://controller.tar.gz", + ControllerVersion: "v2.0.0", + NftablesExporterURL: "http://exporter.tar.gz", + NftablesExporterVersion: "v1.0.0", + Interval: defaults.DefaultFirewallReconcileInterval, }, - }, - Spec: v2.FirewallSpec{ - Size: "n1-medium-x86", - Project: "project-a", - Partition: "partition-a", - Image: "firewall-ubuntu-2.0", - Networks: []string{"internet"}, - ControllerURL: "http://controller.tar.gz", - ControllerVersion: "v2.0.0", - NftablesExporterURL: "http://exporter.tar.gz", - NftablesExporterVersion: "v1.0.0", - Interval: defaults.DefaultFirewallReconcileInterval, }, }, - }, + } } ) @@ -154,60 +159,47 @@ var _ = Context("integration test", Ordered, func() { Expect(client.IgnoreAlreadyExists(k8sClient.Create(ctx, sshSecret.DeepCopy()))).To(Succeed()) Expect(client.IgnoreAlreadyExists(k8sClient.Create(ctx, genericKubeconfigSecret(apiCA, apiHost, apiCert, apiKey)))).To(Succeed()) Expect(client.IgnoreAlreadyExists(k8sClient.Create(ctx, shootTokenSecret.DeepCopy()))).To(Succeed()) - DeferCleanup(func() { - swapMetalClient(&metalclient.MetalMockFns{ - Firewall: func(m *mock.Mock) { - m.On("AllocateFirewall", mock.Anything, nil).Return(&metalfirewall.AllocateFirewallOK{Payload: firewall1}, nil).Maybe() - m.On("FindFirewalls", mock.Anything, nil).Return(&metalfirewall.FindFirewallsOK{Payload: []*models.V1FirewallResponse{firewall1}}, nil).Maybe() - }, - Network: func(m *mock.Mock) { - m.On("FindNetwork", mock.Anything, nil).Return(&network.FindNetworkOK{Payload: network1}, nil).Maybe() - }, - Machine: func(m *mock.Mock) { - m.On("FreeMachine", mock.Anything, nil).Return(&machine.FreeMachineOK{Payload: &models.V1MachineResponse{ID: firewall1.ID}}, nil).Maybe() - m.On("UpdateMachine", mock.Anything, nil).Return(&machine.UpdateMachineOK{Payload: &models.V1MachineResponse{}}, nil).Maybe() - }, - }) - Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, deployment.DeepCopy()))).To(Succeed()) - _ = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 0, &v2.FirewallDeploymentList{}, func(l *v2.FirewallDeploymentList) []*v2.FirewallDeployment { - return l.GetItems() - }, 10*time.Second) - }) }) - Describe("the good case", Ordered, func() { - swapMetalClient(&metalclient.MetalMockFns{ - Firewall: func(m *mock.Mock) { - m.On("AllocateFirewall", mock.Anything, nil).Return(&metalfirewall.AllocateFirewallOK{Payload: firewall1}, nil).Maybe() - m.On("FindFirewalls", mock.Anything, nil).Return(&metalfirewall.FindFirewallsOK{Payload: []*models.V1FirewallResponse{firewall1}}, nil).Maybe() - }, - Network: func(m *mock.Mock) { - m.On("FindNetwork", mock.Anything, nil).Return(&network.FindNetworkOK{Payload: network1}, nil).Maybe() - }, - Machine: func(m *mock.Mock) { - m.On("UpdateMachine", mock.Anything, nil).Return(&machine.UpdateMachineOK{Payload: &models.V1MachineResponse{}}, nil).Maybe() - }, - }) - - var ( - fw *v2.Firewall - set *v2.FirewallSet - mon *v2.FirewallMonitor - ) - + Describe("the rolling update", Ordered, func() { When("creating a firewall deployment", Ordered, func() { It("the creation works", func() { - Expect(k8sClient.Create(ctx, deployment)).To(Succeed()) + swapMetalClient(&metalclient.MetalMockFns{ + Firewall: func(m *mock.Mock) { + m.On("AllocateFirewall", mock.Anything, nil).Return(&metalfirewall.AllocateFirewallOK{Payload: firewall1}, nil).Maybe() + m.On("FindFirewalls", mock.Anything, nil).Return(&metalfirewall.FindFirewallsOK{Payload: []*models.V1FirewallResponse{firewall1}}, nil).Maybe() + }, + Network: func(m *mock.Mock) { + m.On("FindNetwork", mock.Anything, nil).Return(&network.FindNetworkOK{Payload: network1}, nil).Maybe() + }, + Machine: func(m *mock.Mock) { + m.On("UpdateMachine", mock.Anything, nil).Return(&machine.UpdateMachineOK{Payload: &models.V1MachineResponse{}}, nil).Maybe() + }, + }) + + Expect(k8sClient.Create(ctx, deployment())).To(Succeed()) }) It("the userdata was rendered by the defaulting webhook", func() { - deploy := deployment.DeepCopy() - Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(deployment), deploy)).To(Succeed()) + deploy := &v2.FirewallDeployment{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(deployment()), deploy)).To(Succeed()) Expect(deploy.Spec.Template.Spec.Userdata).NotTo(BeEmpty()) }) + + It("the update strategy is rolling update", func() { + deploy := &v2.FirewallDeployment{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(deployment()), deploy)).To(Succeed()) + Expect(deploy.Spec.Strategy).To(Equal(v2.StrategyRollingUpdate)) + }) }) - Describe("new resources will be spawned by the controller", func() { + Describe("new resources will be spawned by the controller", Ordered, func() { + var ( + fw *v2.Firewall + set *v2.FirewallSet + mon *v2.FirewallMonitor + ) + It("should create a firewall set", func() { set = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 1, &v2.FirewallSetList{}, func(l *v2.FirewallSetList) []*v2.FirewallSet { return l.GetItems() @@ -236,684 +228,1371 @@ var _ = Context("integration test", Ordered, func() { } Expect(k8sClient.Update(ctx, mon)).To(Succeed()) }) - }) - Context("the firewall resource", func() { - It("should be named after the namespace (it's the shoot name in the end)", func() { - Expect(fw.Name).To(HavePrefix(namespaceName + "-firewall-")) - }) + Context("the firewall resource", func() { + It("should be named after the namespace (it's the shoot name in the end)", func() { + Expect(fw.Name).To(HavePrefix(namespaceName + "-firewall-")) + }) - It("should be in the same namespace as the set", func() { - Expect(fw.Namespace).To(Equal(set.Namespace)) - }) + It("should be in the same namespace as the set", func() { + Expect(fw.Namespace).To(Equal(set.Namespace)) + }) - It("should inherit the spec from the set", func() { - wantSpec := set.Spec.Template.Spec.DeepCopy() - Expect(&fw.Spec).To(BeComparableTo(wantSpec)) - }) + It("should inherit the spec from the set", func() { + wantSpec := set.Spec.Template.Spec.DeepCopy() + Expect(&fw.Spec).To(BeComparableTo(wantSpec)) + }) - It("should have the set as an owner", func() { - Expect(fw.ObjectMeta.OwnerReferences).To(HaveLen(1)) - Expect(fw.ObjectMeta.OwnerReferences[0].Name).To(Equal(set.Name)) - }) + It("should have the set as an owner", func() { + Expect(fw.ObjectMeta.OwnerReferences).To(HaveLen(1)) + Expect(fw.ObjectMeta.OwnerReferences[0].Name).To(Equal(set.Name)) + }) - It("should have the created condition true", func() { - cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { - return fd.Status.Conditions - }, v2.FirewallCreated, v2.ConditionTrue, 15*time.Second) + It("should have the created condition true", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallCreated, v2.ConditionTrue, 15*time.Second) - Expect(cond.LastTransitionTime).NotTo(BeZero()) - Expect(cond.LastUpdateTime).NotTo(BeZero()) - Expect(cond.Reason).To(Equal("Created")) - Expect(cond.Message).To(Equal(fmt.Sprintf("Firewall %q created successfully.", *firewall1.Allocation.Name))) - }) + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Equal("Created")) + Expect(cond.Message).To(Equal(fmt.Sprintf("Firewall %q created successfully.", *firewall1.Allocation.Name))) + }) - It("should populate the machine status", func() { - var status *v2.MachineStatus - var fw = fw.DeepCopy() - Eventually(func() *v2.MachineStatus { - Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(fw), fw)).To(Succeed()) - status = fw.Status.MachineStatus - return status - }, 5*time.Second, interval).Should(Not(BeNil())) + It("should populate the machine status", func() { + var status *v2.MachineStatus + var fw = fw.DeepCopy() + Eventually(func() *v2.MachineStatus { + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(fw), fw)).To(Succeed()) + status = fw.Status.MachineStatus + return status + }, 5*time.Second, interval).Should(Not(BeNil())) + + Expect(status.MachineID).To(Equal(*firewall1.ID)) + Expect(status.CrashLoop).To(Equal(false)) + Expect(status.Liveliness).To(Equal("Alive")) + Expect(status.LastEvent).NotTo(BeNil()) + Expect(status.LastEvent.Event).To(Equal("Phoned Home")) + Expect(status.LastEvent.Message).To(Equal("phoning home")) + }) - Expect(status.MachineID).To(Equal(*firewall1.ID)) - Expect(status.CrashLoop).To(Equal(false)) - Expect(status.Liveliness).To(Equal("Alive")) - Expect(status.LastEvent).NotTo(BeNil()) - Expect(status.LastEvent.Event).To(Equal("Phoned Home")) - Expect(status.LastEvent.Message).To(Equal("phoning home")) - }) + It("should have the ready condition true", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallReady, v2.ConditionTrue, 15*time.Second) - It("should have the ready condition true", func() { - cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { - return fd.Status.Conditions - }, v2.FirewallReady, v2.ConditionTrue, 15*time.Second) + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Equal("Ready")) + Expect(cond.Message).To(Equal(fmt.Sprintf("Firewall %q is phoning home and alive.", *firewall1.Allocation.Name))) + }) - Expect(cond.LastTransitionTime).NotTo(BeZero()) - Expect(cond.LastUpdateTime).NotTo(BeZero()) - Expect(cond.Reason).To(Equal("Ready")) - Expect(cond.Message).To(Equal(fmt.Sprintf("Firewall %q is phoning home and alive.", *firewall1.Allocation.Name))) - }) + It("should have the monitor condition true", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallMonitorDeployed, v2.ConditionTrue, 5*time.Second) - It("should have the monitor condition true", func() { - cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { - return fd.Status.Conditions - }, v2.FirewallMonitorDeployed, v2.ConditionTrue, 5*time.Second) + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Equal("Deployed")) + Expect(cond.Message).To(Equal("Successfully deployed firewall-monitor.")) + }) - Expect(cond.LastTransitionTime).NotTo(BeZero()) - Expect(cond.LastUpdateTime).NotTo(BeZero()) - Expect(cond.Reason).To(Equal("Deployed")) - Expect(cond.Message).To(Equal("Successfully deployed firewall-monitor.")) - }) + It("should have the firewall-controller connected condition true", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallControllerConnected, v2.ConditionTrue, 15*time.Second) - It("should have the firewall-controller connected condition true", func() { - cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { - return fd.Status.Conditions - }, v2.FirewallControllerConnected, v2.ConditionTrue, 15*time.Second) + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Equal("Connected")) + Expect(cond.Message).To(Equal(fmt.Sprintf("Controller reconciled firewall at %s.", mon.ControllerStatus.Updated.Time.String()))) + }) - Expect(cond.LastTransitionTime).NotTo(BeZero()) - Expect(cond.LastUpdateTime).NotTo(BeZero()) - Expect(cond.Reason).To(Equal("Connected")) - Expect(cond.Message).To(Equal(fmt.Sprintf("Controller reconciled firewall at %s.", mon.ControllerStatus.Updated.Time.String()))) + It("should have configured the distance", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallDistanceConfigured, v2.ConditionTrue, 15*time.Second) + + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Equal("Configured")) + Expect(cond.Message).To(Equal(fmt.Sprintf("Controller has configured the specified distance %d.", v2.FirewallShortestDistance))) + }) }) - It("should have configured the distance", func() { - cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { - return fd.Status.Conditions - }, v2.FirewallDistanceConfigured, v2.ConditionTrue, 15*time.Second) + Context("the firewall set resource", func() { + It("should be named after the deployment", func() { + Expect(set.Name).To(HavePrefix(deployment().Name + "-")) + }) - Expect(cond.LastTransitionTime).NotTo(BeZero()) - Expect(cond.LastUpdateTime).NotTo(BeZero()) - Expect(cond.Reason).To(Equal("Configured")) - Expect(cond.Message).To(Equal(fmt.Sprintf("Controller has configured the specified distance %d.", v2.FirewallShortestDistance))) - }) - }) + It("should be in the same namespace as the deployment", func() { + Expect(set.Namespace).To(Equal(deployment().Namespace)) + }) - Context("the firewall set resource", func() { - It("should be named after the deployment", func() { - Expect(set.Name).To(HavePrefix(deployment.Name + "-")) - }) + It("should take the same replicas as defined by the deployment", func() { + Expect(set.Spec.Replicas).To(Equal(1)) + }) - It("should be in the same namespace as the deployment", func() { - Expect(set.Namespace).To(Equal(deployment.Namespace)) - }) + It("should inherit the spec from the deployment", func() { + deploy := &v2.FirewallDeployment{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(deployment()), deploy)).To(Succeed()) + Expect(set.Spec.Template.Spec).To(BeComparableTo(deploy.Spec.Template.Spec)) + }) - It("should take the same replicas as defined by the deployement", func() { - Expect(set.Spec.Replicas).To(Equal(1)) - }) + It("should have the deployment as an owner", func() { + Expect(set.ObjectMeta.OwnerReferences).To(HaveLen(1)) + Expect(set.ObjectMeta.OwnerReferences[0].Name).To(Equal(deployment().Name)) + }) - It("should inherit the spec from the deployement", func() { - wantSpec := deployment.Spec.Template.Spec.DeepCopy() - Expect(&set.Spec.Template.Spec).To(BeComparableTo(wantSpec)) + It("should populate the status", func() { + var set = set.DeepCopy() + Eventually(func() int { + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(set), set)).To(Succeed()) + return set.Status.ReadyReplicas + }, 15*time.Second, interval).Should(Equal(1), "reach ready replicas") + + Expect(set.Status.TargetReplicas).To(Equal(1)) + Expect(set.Status.ProgressingReplicas).To(Equal(0)) + Expect(set.Status.UnhealthyReplicas).To(Equal(0)) + Expect(set.Status.ObservedRevision).To(Equal(0)) // this is the first revision + }) }) - It("should have the deployment as an owner", func() { - Expect(set.ObjectMeta.OwnerReferences).To(HaveLen(1)) - Expect(set.ObjectMeta.OwnerReferences[0].Name).To(Equal(deployment.Name)) - }) + Context("the firewall deployment resource", func() { + It("should default the update strategy to rolling update (so the mutating webhook is working)", func() { + deploy := &v2.FirewallDeployment{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(deployment()), deploy)).To(Succeed()) + Expect(deploy.Spec.Strategy).To(Equal(v2.StrategyRollingUpdate)) + }) - It("should populate the status", func() { - var set = set.DeepCopy() - Eventually(func() int { - Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(set), set)).To(Succeed()) - return set.Status.ReadyReplicas - }, 15*time.Second, interval).Should(Equal(1), "reach ready replicas") + It("should have the rbac condition true", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, deployment(), func(fd *v2.FirewallDeployment) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallDeplomentRBACProvisioned, v2.ConditionTrue, 5*time.Second) - Expect(set.Status.TargetReplicas).To(Equal(1)) - Expect(set.Status.ProgressingReplicas).To(Equal(0)) - Expect(set.Status.UnhealthyReplicas).To(Equal(0)) - Expect(set.Status.ObservedRevision).To(Equal(0)) // this is the first revision - }) - }) + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Equal("Provisioned")) + Expect(cond.Message).To(Equal("RBAC provisioned successfully.")) + }) - Context("the firewall deployment resource", func() { - It("should default the update strategy to rolling update (so the mutating webhook is working)", func() { - Expect(deployment.Spec.Strategy).To(Equal(v2.StrategyRollingUpdate)) - }) + It("should have the available condition true", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, deployment(), func(fd *v2.FirewallDeployment) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallDeplomentAvailable, v2.ConditionTrue, 5*time.Second) - It("should have the rbac condition true", func() { - cond := testcommon.WaitForCondition(k8sClient, ctx, deployment.DeepCopy(), func(fd *v2.FirewallDeployment) v2.Conditions { - return fd.Status.Conditions - }, v2.FirewallDeplomentRBACProvisioned, v2.ConditionTrue, 5*time.Second) - - Expect(cond.LastTransitionTime).NotTo(BeZero()) - Expect(cond.LastUpdateTime).NotTo(BeZero()) - Expect(cond.Reason).To(Equal("Provisioned")) - Expect(cond.Message).To(Equal("RBAC provisioned successfully.")) - }) + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Equal("MinimumReplicasAvailable")) + Expect(cond.Message).To(Equal("Deployment has minimum availability.")) + }) - It("should have the available condition true", func() { - cond := testcommon.WaitForCondition(k8sClient, ctx, deployment.DeepCopy(), func(fd *v2.FirewallDeployment) v2.Conditions { - return fd.Status.Conditions - }, v2.FirewallDeplomentAvailable, v2.ConditionTrue, 5*time.Second) + It("should have the progress condition true", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, deployment(), func(fd *v2.FirewallDeployment) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallDeplomentProgressing, v2.ConditionTrue, 15*time.Second) + + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Or(Equal("NewFirewallSetAvailable"), Equal("FirewallSetUpdated"))) + Expect(cond.Message).To(Or( + Equal(fmt.Sprintf("FirewallSet %q has successfully progressed.", set.Name)), + Equal(fmt.Sprintf("Updated firewall set %q.", set.Name)), + )) + }) - Expect(cond.LastTransitionTime).NotTo(BeZero()) - Expect(cond.LastUpdateTime).NotTo(BeZero()) - Expect(cond.Reason).To(Equal("MinimumReplicasAvailable")) - Expect(cond.Message).To(Equal("Deployment has minimum availability.")) + It("should populate the status", func() { + deploy := &v2.FirewallDeployment{} + Eventually(func() int { + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(deployment()), deploy)).To(Succeed()) + return deploy.Status.ReadyReplicas + }, 15*time.Second, interval).Should(Equal(1), "reach ready replicas") + + Expect(deploy.Status.TargetReplicas).To(Equal(1)) + Expect(deploy.Status.ProgressingReplicas).To(Equal(0)) + Expect(deploy.Status.UnhealthyReplicas).To(Equal(0)) + Expect(deploy.Status.ObservedRevision).To(Equal(0)) // this is the first revision + }) }) + }) - It("should have the progress condition true", func() { - cond := testcommon.WaitForCondition(k8sClient, ctx, deployment.DeepCopy(), func(fd *v2.FirewallDeployment) v2.Conditions { - return fd.Status.Conditions - }, v2.FirewallDeplomentProgressing, v2.ConditionTrue, 15*time.Second) + When("a significant change occurs", Ordered, func() { + var ( + installingFirewall = firewall2("Installing", "is installing") + ) + + Context("the spec is updated", func() { + It("the update works", func() { + swapMetalClient(&metalclient.MetalMockFns{ + Firewall: func(m *mock.Mock) { + m.On("AllocateFirewall", mock.Anything, nil).Return(&metalfirewall.AllocateFirewallOK{Payload: installingFirewall}, nil).Maybe() + m.On("FindFirewalls", mock.Anything, nil).Return(&metalfirewall.FindFirewallsOK{Payload: []*models.V1FirewallResponse{installingFirewall}}, nil).Maybe() + }, + Network: func(m *mock.Mock) { + m.On("FindNetwork", mock.Anything, nil).Return(&network.FindNetworkOK{Payload: network1}, nil).Maybe() + }, + Machine: func(m *mock.Mock) { + m.On("UpdateMachine", mock.Anything, nil).Return(&machine.UpdateMachineOK{Payload: &models.V1MachineResponse{}}, nil).Maybe() + }, + }) - Expect(cond.LastTransitionTime).NotTo(BeZero()) - Expect(cond.LastUpdateTime).NotTo(BeZero()) - Expect(cond.Reason).To(Or(Equal("NewFirewallSetAvailable"), Equal("FirewallSetUpdated"))) - Expect(cond.Message).To(Or( - Equal(fmt.Sprintf("FirewallSet %q has successfully progressed.", set.Name)), - Equal(fmt.Sprintf("Updated firewall set %q.", set.Name)), - )) - }) + deploy := deployment() + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(deployment()), deploy)).To(Succeed()) - It("should populate the status", func() { - var deploy = deployment.DeepCopy() - Eventually(func() int { - Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(deploy), deploy)).To(Succeed()) - return deploy.Status.ReadyReplicas - }, 15*time.Second, interval).Should(Equal(1), "reach ready replicas") + deploy.Spec.Template.Spec.Size = "n2-medium-x86" - Expect(deploy.Status.TargetReplicas).To(Equal(1)) - Expect(deploy.Status.ProgressingReplicas).To(Equal(0)) - Expect(deploy.Status.UnhealthyReplicas).To(Equal(0)) - Expect(deploy.Status.ObservedRevision).To(Equal(0)) // this is the first revision + Expect(k8sClient.Update(ctx, deploy)).To(Succeed()) + }) }) - }) - }) - Describe("the rolling update", Ordered, func() { - var ( - installingFirewall = firewall2("Installing", "is installing") + var ( + fw *v2.Firewall + set *v2.FirewallSet + mon *v2.FirewallMonitor + ) - deploy = deployment.DeepCopy() - fw *v2.Firewall - set *v2.FirewallSet - mon *v2.FirewallMonitor - ) + Context("new resources will be spawned by the controller", func() { - When("updating a significant field that triggers a rolling update", func() { - It("the update works", func() { - Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(deployment), deploy)).To(Succeed()) - - swapMetalClient(&metalclient.MetalMockFns{ - Firewall: func(m *mock.Mock) { - m.On("AllocateFirewall", mock.Anything, nil).Return(&metalfirewall.AllocateFirewallOK{Payload: installingFirewall}, nil).Maybe() - m.On("FindFirewalls", mock.Anything, nil).Return(&metalfirewall.FindFirewallsOK{Payload: []*models.V1FirewallResponse{installingFirewall}}, nil).Maybe() - }, - Network: func(m *mock.Mock) { - m.On("FindNetwork", mock.Anything, nil).Return(&network.FindNetworkOK{Payload: network1}, nil).Maybe() - }, - Machine: func(m *mock.Mock) { - m.On("UpdateMachine", mock.Anything, nil).Return(&machine.UpdateMachineOK{Payload: &models.V1MachineResponse{}}, nil).Maybe() - }, + It("should create another firewall set", func() { + set = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 2, &v2.FirewallSetList{}, func(l *v2.FirewallSetList) []*v2.FirewallSet { + return l.GetItems() + }, 15*time.Second) }) - deploy.Spec.Template.Spec.Size = "n2-medium-x86" + It("should create another firewall", func() { + fw = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 2, &v2.FirewallList{}, func(l *v2.FirewallList) []*v2.Firewall { + return l.GetItems() + }, 15*time.Second) + }) - Expect(k8sClient.Update(ctx, deploy)).To(Succeed()) - }) - }) + It("should create another firewall monitor", func() { + mon = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 2, &v2.FirewallMonitorList{}, func(l *v2.FirewallMonitorList) []*v2.FirewallMonitor { + return l.GetItems() + }, 15*time.Second) + }) - Context("new resources will be spawned by the controller", func() { - It("should create another firewall set", func() { - set = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 2, &v2.FirewallSetList{}, func(l *v2.FirewallSetList) []*v2.FirewallSet { - return l.GetItems() - }, 15*time.Second) - }) + Context("the new firewall resource", func() { + It("should be named after the namespace (it's the shoot name in the end)", func() { + Expect(fw.Name).To(HavePrefix(namespaceName + "-firewall-")) + }) + + It("should be in the same namespace as the set", func() { + Expect(fw.Namespace).To(Equal(set.Namespace)) + }) + + It("should inherit the spec from the set", func() { + wantSpec := set.Spec.Template.Spec.DeepCopy() + Expect(&fw.Spec).To(BeComparableTo(wantSpec)) + }) + + It("should have the set as an owner", func() { + Expect(fw.ObjectMeta.OwnerReferences).To(HaveLen(1)) + Expect(fw.ObjectMeta.OwnerReferences[0].Name).To(Equal(set.Name)) + }) + + It("should have the created condition true", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallCreated, v2.ConditionTrue, 15*time.Second) + + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Equal("Created")) + Expect(cond.Message).To(Equal(fmt.Sprintf("Firewall %q created successfully.", *installingFirewall.Allocation.Name))) + }) + + It("should populate the machine status", func() { + var status *v2.MachineStatus + var fw = fw.DeepCopy() + Eventually(func() *v2.MachineStatus { + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(fw), fw)).To(Succeed()) + status = fw.Status.MachineStatus + return status + }, 5*time.Second, interval).Should(Not(BeNil())) + + Expect(status.MachineID).To(Equal(*installingFirewall.ID)) + Expect(status.CrashLoop).To(Equal(false)) + Expect(status.Liveliness).To(Equal("Alive")) + Expect(status.LastEvent).NotTo(BeNil()) + Expect(status.LastEvent.Event).To(Equal("Installing")) + Expect(status.LastEvent.Message).To(Equal("is installing")) + }) + + It("should have the ready condition false", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallReady, v2.ConditionFalse, 15*time.Second) + + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Equal("NotReady")) + Expect(cond.Message).To(Equal(fmt.Sprintf("Firewall %q is not ready.", *installingFirewall.Allocation.Name))) + }) + + It("should not yet have a distance configured", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallDistanceConfigured, v2.ConditionFalse, 15*time.Second) + + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Equal("NotConnected")) + Expect(cond.Message).To(Equal("Controller has not yet connected.")) + }) + + It("should have the monitor condition true", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallMonitorDeployed, v2.ConditionTrue, 5*time.Second) + + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Equal("Deployed")) + Expect(cond.Message).To(Equal("Successfully deployed firewall-monitor.")) + }) + + It("should have the firewall-controller connected condition false", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallControllerConnected, v2.ConditionFalse, 15*time.Second) + + Expect(cond.LastTransitionTime).NotTo(BeZero()) + }) + + It("should have firewall networks populated", func() { + var nws []v2.FirewallNetwork + var fw = fw.DeepCopy() + Eventually(func() []v2.FirewallNetwork { + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(fw), fw)).To(Succeed()) + nws = fw.Status.FirewallNetworks + return nws + }, 5*time.Second, interval).Should(HaveLen(1)) + + Expect(nws).To(BeComparableTo([]v2.FirewallNetwork{ + { + ASN: installingFirewall.Allocation.Networks[0].Asn, + DestinationPrefixes: installingFirewall.Allocation.Networks[0].Destinationprefixes, + IPs: installingFirewall.Allocation.Networks[0].Ips, + Nat: installingFirewall.Allocation.Networks[0].Nat, + NetworkID: installingFirewall.Allocation.Networks[0].Networkid, + NetworkType: installingFirewall.Allocation.Networks[0].Networktype, + Prefixes: network1.Prefixes, + Vrf: installingFirewall.Allocation.Networks[0].Vrf, + }, + })) + }) + + It("should have shoot access populated", func() { + var access *v2.ShootAccess + var fw = fw.DeepCopy() + Eventually(func() *v2.ShootAccess { + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(fw), fw)).To(Succeed()) + access = fw.Status.ShootAccess + return access + }, 5*time.Second, interval).Should(Not(BeNil())) + + Expect(access).To(BeComparableTo(&v2.ShootAccess{ + GenericKubeconfigSecretName: "kubeconfig-secret-name", + TokenSecretName: "token", + Namespace: namespaceName, + APIServerURL: apiHost, + SSHKeySecretName: sshSecret.Name, + })) + }) + }) - It("should create another firewall", func() { - fw = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 2, &v2.FirewallList{}, func(l *v2.FirewallList) []*v2.Firewall { - return l.GetItems() - }, 15*time.Second) - }) + Context("the new firewall set resource", func() { + It("should be named after the deployment", func() { + Expect(set.Name).To(HavePrefix(deployment().Name + "-")) + }) + + It("should be in the same namespace as the deployment", func() { + Expect(set.Namespace).To(Equal(deployment().Namespace)) + }) + + It("should take the same replicas as defined by the deployment", func() { + Expect(set.Spec.Replicas).To(Equal(1)) + }) + + It("should inherit the spec from the deployment", func() { + deploy := &v2.FirewallDeployment{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(deployment()), deploy)).To(Succeed()) + + wantSpec := deploy.Spec.Template.Spec.DeepCopy() + wantSpec.Size = "n2-medium-x86" // this is the change that triggered the rolling update + Expect(&set.Spec.Template.Spec).To(BeComparableTo(wantSpec)) + }) + + It("should start with a higher distance", func() { + Expect(set.Spec.Distance).To(Equal(v2.FirewallRollingUpdateSetDistance)) + }) + + It("should have the deployment as an owner", func() { + Expect(set.ObjectMeta.OwnerReferences).To(HaveLen(1)) + Expect(set.ObjectMeta.OwnerReferences[0].Name).To(Equal(deployment().Name)) + }) + + It("should populate the status", func() { + var set = set.DeepCopy() + Eventually(func() int { + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(set), set)).To(Succeed()) + return set.Status.ProgressingReplicas + }, 15*time.Second, interval).Should(Equal(1), "reach progressing replicas") + + Expect(set.Status.TargetReplicas).To(Equal(1)) + Expect(set.Status.ReadyReplicas).To(Equal(0)) + Expect(set.Status.UnhealthyReplicas).To(Equal(0)) + Expect(set.Status.ObservedRevision).To(Equal(1)) + }) + }) - It("should create another firewall monitor", func() { - mon = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 2, &v2.FirewallMonitorList{}, func(l *v2.FirewallMonitorList) []*v2.FirewallMonitor { - return l.GetItems() - }, 15*time.Second) - }) - }) + Context("the firewall deployment resource", func() { + It("should have the rbac condition true", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, deployment(), func(fd *v2.FirewallDeployment) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallDeplomentRBACProvisioned, v2.ConditionTrue, 5*time.Second) + + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Equal("Provisioned")) + Expect(cond.Message).To(Equal("RBAC provisioned successfully.")) + }) + + It("should have the available condition false", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, deployment(), func(fd *v2.FirewallDeployment) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallDeplomentAvailable, v2.ConditionFalse, 5*time.Second) + + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Equal("MinimumReplicasUnavailable")) + Expect(cond.Message).To(Equal("Deployment does not have minimum availability.")) + }) + + It("should have the progress condition true", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, deployment(), func(fd *v2.FirewallDeployment) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallDeplomentProgressing, v2.ConditionTrue, 15*time.Second) + + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Or(Equal("NewFirewallSetAvailable"), Equal("FirewallSetUpdated"))) + Expect(cond.Message).To(Or( + Equal(fmt.Sprintf("FirewallSet %q has successfully progressed.", set.Name)), + Equal(fmt.Sprintf("Updated firewall set %q.", set.Name)), + )) + }) + + It("should populate the status", func() { + deploy := &v2.FirewallDeployment{} + Eventually(func() int { + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(deployment()), deploy)).To(Succeed()) + return deploy.Status.ProgressingReplicas + }, 15*time.Second, interval).Should(Equal(1), "reach progressing replicas") + + Expect(deploy.Status.TargetReplicas).To(Equal(1)) + Expect(deploy.Status.ReadyReplicas).To(Equal(0)) + Expect(deploy.Status.UnhealthyReplicas).To(Equal(0)) + Expect(deploy.Status.ObservedRevision).To(Equal(1)) + }) + + It("should not be possible to update deployment strategy while deployment has not converged", func() { + deploy := &v2.FirewallDeployment{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(deployment()), deploy)).To(Succeed()) + deploy.Spec.Strategy = v2.StrategyRecreate + err := k8sClient.Update(ctx, deploy) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(`Invalid value: "Recreate": strategy can not be updated until target replicas have been reached (i.e. deployment has converged)`)) + }) + }) - Context("the new firewall resource", func() { - It("should be named after the namespace (it's the shoot name in the end)", func() { - Expect(fw.Name).To(HavePrefix(namespaceName + "-firewall-")) + // TODO: verify the monitor resource }) - It("should be in the same namespace as the set", func() { - Expect(fw.Namespace).To(Equal(set.Namespace)) - }) + var ( + readyFirewall = firewall2("Phoned Home", "is phoning home") + ) - It("should inherit the spec from the set", func() { - wantSpec := set.Spec.Template.Spec.DeepCopy() - Expect(&fw.Spec).To(BeComparableTo(wantSpec)) - }) + When("the firewall gets ready and the firewall-controller connects", Ordered, func() { + It("should allow an update of the firewall monitor", func() { + swapMetalClient(&metalclient.MetalMockFns{ + Machine: func(m *mock.Mock) { + m.On("FreeMachine", mock.Anything, nil).Return(&machine.FreeMachineOK{Payload: &models.V1MachineResponse{ID: firewall1.ID}}, nil).Maybe() + m.On("UpdateMachine", mock.Anything, nil).Return(&machine.UpdateMachineOK{Payload: &models.V1MachineResponse{}}, nil).Maybe() + }, + Firewall: func(m *mock.Mock) { + m.On("FindFirewalls", mock.Anything, nil).Return(&metalfirewall.FindFirewallsOK{Payload: []*models.V1FirewallResponse{readyFirewall}}, nil).Maybe() + }, + Network: func(m *mock.Mock) { + m.On("FindNetwork", mock.Anything, nil).Return(&network.FindNetworkOK{Payload: network1}, nil).Maybe() + }, + }) + + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(mon), mon)).To(Succeed()) // refetch + // simulating a firewall-controller updating the resource + mon.ControllerStatus = &v2.ControllerStatus{ + Updated: metav1.NewTime(time.Now()), + Distance: v2.FirewallRollingUpdateSetDistance, + DistanceSupported: true, + } + Expect(k8sClient.Update(ctx, mon)).To(Succeed()) + }) - It("should have the set as an owner", func() { - Expect(fw.ObjectMeta.OwnerReferences).To(HaveLen(1)) - Expect(fw.ObjectMeta.OwnerReferences[0].Name).To(Equal(set.Name)) - }) + It("the firewall-controller reflects the distance during a rolling update", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallDistanceConfigured, v2.ConditionTrue, 15*time.Second) - It("should have the created condition true", func() { - cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { - return fd.Status.Conditions - }, v2.FirewallCreated, v2.ConditionTrue, 15*time.Second) + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Equal("Configured")) + Expect(cond.Message).To(Equal(fmt.Sprintf("Controller has configured the specified distance %d.", v2.FirewallRollingUpdateSetDistance))) + }) - Expect(cond.LastTransitionTime).NotTo(BeZero()) - Expect(cond.LastUpdateTime).NotTo(BeZero()) - Expect(cond.Reason).To(Equal("Created")) - Expect(cond.Message).To(Equal(fmt.Sprintf("Firewall %q created successfully.", *installingFirewall.Allocation.Name))) + Context("the old generation disappears", Ordered, func() { + var ( + fw *v2.Firewall + set *v2.FirewallSet + ) + + It("should delete firewall monitor", func() { + mon := testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 1, &v2.FirewallMonitorList{}, func(l *v2.FirewallMonitorList) []*v2.FirewallMonitor { + return l.GetItems() + }, 15*time.Second) + Expect(mon.MachineStatus.MachineID).To(Equal(*readyFirewall.ID)) + }) + + It("should delete the firewall", func() { + fw = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 1, &v2.FirewallList{}, func(l *v2.FirewallList) []*v2.Firewall { + return l.GetItems() + }, 15*time.Second) + Expect(fw.Status.MachineStatus.MachineID).To(Equal(*readyFirewall.ID)) + }) + + It("should delete the firewall set", func() { + set = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 1, &v2.FirewallSetList{}, func(l *v2.FirewallSetList) []*v2.FirewallSet { + return l.GetItems() + }, 15*time.Second) + Expect(set.Status.ObservedRevision).To(Equal(1)) + }) + + Context("the update is finalizes", func() { + It("should populate the controller status field in the firewall resource", func() { + var fw = fw.DeepCopy() + Eventually(func() *v2.ControllerConnection { + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(fw), fw)).To(Succeed()) + return fw.Status.ControllerStatus + }, 15*time.Second, interval).Should(Not(BeNil()), "controller connection was not synced to firewall resource") + }) + + It("the firewall set should be updated to shortest distance as the update has succeeded", func() { + Eventually(func() v2.FirewallDistance { + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(set), set)).To(Succeed()) + return set.Spec.Distance + }).Within(5 * time.Second).ProbeEvery(interval).Should(Equal(v2.FirewallShortestDistance)) + Expect(set.Spec.Distance).To(Equal(v2.FirewallShortestDistance)) + }) + }) + }) }) + }) - It("should populate the machine status", func() { - var status *v2.MachineStatus - var fw = fw.DeepCopy() - Eventually(func() *v2.MachineStatus { - Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(fw), fw)).To(Succeed()) - status = fw.Status.MachineStatus - return status - }, 5*time.Second, interval).Should(Not(BeNil())) + Describe("the deletion flow", Ordered, func() { + When("deleting the firewall deployment", func() { + It("the deletion finishes", func() { + swapMetalClient(&metalclient.MetalMockFns{ + Firewall: func(m *mock.Mock) { + m.On("AllocateFirewall", mock.Anything, nil).Return(&metalfirewall.AllocateFirewallOK{Payload: firewall1}, nil).Maybe() + m.On("FindFirewalls", mock.Anything, nil).Return(&metalfirewall.FindFirewallsOK{Payload: []*models.V1FirewallResponse{firewall1}}, nil).Maybe() + }, + Network: func(m *mock.Mock) { + m.On("FindNetwork", mock.Anything, nil).Return(&network.FindNetworkOK{Payload: network1}, nil).Maybe() + }, + Machine: func(m *mock.Mock) { + m.On("FreeMachine", mock.Anything, nil).Return(&machine.FreeMachineOK{Payload: &models.V1MachineResponse{ID: firewall1.ID}}, nil).Maybe() + m.On("UpdateMachine", mock.Anything, nil).Return(&machine.UpdateMachineOK{Payload: &models.V1MachineResponse{}}, nil).Maybe() + }, + }) - Expect(status.MachineID).To(Equal(*installingFirewall.ID)) - Expect(status.CrashLoop).To(Equal(false)) - Expect(status.Liveliness).To(Equal("Alive")) - Expect(status.LastEvent).NotTo(BeNil()) - Expect(status.LastEvent.Event).To(Equal("Installing")) - Expect(status.LastEvent.Message).To(Equal("is installing")) + Expect(k8sClient.Delete(ctx, deployment())).To(Succeed()) + }) }) - It("should have the ready condition false", func() { - cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { - return fd.Status.Conditions - }, v2.FirewallReady, v2.ConditionFalse, 15*time.Second) - - Expect(cond.LastTransitionTime).NotTo(BeZero()) - Expect(cond.LastUpdateTime).NotTo(BeZero()) - Expect(cond.Reason).To(Equal("NotReady")) - Expect(cond.Message).To(Equal(fmt.Sprintf("Firewall %q is not ready.", *installingFirewall.Allocation.Name))) - }) + Context("all resources are cleaned up", func() { + It("should delete the firewall set", func() { + _ = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 0, &v2.FirewallSetList{}, func(l *v2.FirewallSetList) []*v2.FirewallSet { + return l.GetItems() + }, 10*time.Second) + }) - It("should not yet have a distance configured", func() { - cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { - return fd.Status.Conditions - }, v2.FirewallDistanceConfigured, v2.ConditionFalse, 15*time.Second) + It("should delete the firewall", func() { + _ = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 0, &v2.FirewallList{}, func(l *v2.FirewallList) []*v2.Firewall { + return l.GetItems() + }, 10*time.Second) + }) - Expect(cond.LastTransitionTime).NotTo(BeZero()) - Expect(cond.LastUpdateTime).NotTo(BeZero()) - Expect(cond.Reason).To(Equal("NotConnected")) - Expect(cond.Message).To(Equal("Controller has not yet connected.")) + It("should delete firewall monitor", func() { + _ = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 0, &v2.FirewallMonitorList{}, func(l *v2.FirewallMonitorList) []*v2.FirewallMonitor { + return l.GetItems() + }, 10*time.Second) + }) }) + }) + }) - It("should have the monitor condition true", func() { - cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { - return fd.Status.Conditions - }, v2.FirewallMonitorDeployed, v2.ConditionTrue, 5*time.Second) + Describe("the recreate update", Ordered, func() { + When("creating a firewall deployment", Ordered, func() { + It("the creation works", func() { + swapMetalClient(&metalclient.MetalMockFns{ + Firewall: func(m *mock.Mock) { + m.On("AllocateFirewall", mock.Anything, nil).Return(&metalfirewall.AllocateFirewallOK{Payload: firewall1}, nil).Maybe() + m.On("FindFirewalls", mock.Anything, nil).Return(&metalfirewall.FindFirewallsOK{Payload: []*models.V1FirewallResponse{firewall1}}, nil).Maybe() + }, + Network: func(m *mock.Mock) { + m.On("FindNetwork", mock.Anything, nil).Return(&network.FindNetworkOK{Payload: network1}, nil).Maybe() + }, + Machine: func(m *mock.Mock) { + m.On("UpdateMachine", mock.Anything, nil).Return(&machine.UpdateMachineOK{Payload: &models.V1MachineResponse{}}, nil).Maybe() + }, + }) - Expect(cond.LastTransitionTime).NotTo(BeZero()) - Expect(cond.LastUpdateTime).NotTo(BeZero()) - Expect(cond.Reason).To(Equal("Deployed")) - Expect(cond.Message).To(Equal("Successfully deployed firewall-monitor.")) + deploy := deployment() + deploy.Spec.Strategy = v2.StrategyRecreate + Expect(k8sClient.Create(ctx, deploy)).To(Succeed()) }) - It("should have the firewall-controller connected condition false", func() { - cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { - return fd.Status.Conditions - }, v2.FirewallControllerConnected, v2.ConditionFalse, 15*time.Second) - - Expect(cond.LastTransitionTime).NotTo(BeZero()) + It("the userdata was rendered by the defaulting webhook", func() { + deploy := &v2.FirewallDeployment{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(deployment()), deploy)).To(Succeed()) + Expect(deploy.Spec.Template.Spec.Userdata).NotTo(BeEmpty()) }) - It("should have firewall networks populated", func() { - var nws []v2.FirewallNetwork - var fw = fw.DeepCopy() - Eventually(func() []v2.FirewallNetwork { - Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(fw), fw)).To(Succeed()) - nws = fw.Status.FirewallNetworks - return nws - }, 5*time.Second, interval).Should(HaveLen(1)) - - Expect(nws).To(BeComparableTo([]v2.FirewallNetwork{ - { - ASN: installingFirewall.Allocation.Networks[0].Asn, - DestinationPrefixes: installingFirewall.Allocation.Networks[0].Destinationprefixes, - IPs: installingFirewall.Allocation.Networks[0].Ips, - Nat: installingFirewall.Allocation.Networks[0].Nat, - NetworkID: installingFirewall.Allocation.Networks[0].Networkid, - NetworkType: installingFirewall.Allocation.Networks[0].Networktype, - Prefixes: network1.Prefixes, - Vrf: installingFirewall.Allocation.Networks[0].Vrf, - }, - })) + It("the update strategy is recreate", func() { + deploy := &v2.FirewallDeployment{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(deployment()), deploy)).To(Succeed()) + Expect(deploy.Spec.Strategy).To(Equal(v2.StrategyRecreate)) }) + }) - It("should have shoot access populated", func() { - var access *v2.ShootAccess - var fw = fw.DeepCopy() - Eventually(func() *v2.ShootAccess { - Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(fw), fw)).To(Succeed()) - access = fw.Status.ShootAccess - return access - }, 5*time.Second, interval).Should(Not(BeNil())) + Describe("new resources will be spawned by the controller", Ordered, func() { + var ( + fw *v2.Firewall + set *v2.FirewallSet + mon *v2.FirewallMonitor + ) - Expect(access).To(BeComparableTo(&v2.ShootAccess{ - GenericKubeconfigSecretName: "kubeconfig-secret-name", - TokenSecretName: "token", - Namespace: namespaceName, - APIServerURL: apiHost, - SSHKeySecretName: sshSecret.Name, - })) + It("should create a firewall set", func() { + set = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 1, &v2.FirewallSetList{}, func(l *v2.FirewallSetList) []*v2.FirewallSet { + return l.GetItems() + }, 15*time.Second) }) - }) - Context("the new firewall set resource", func() { - It("should be named after the deployment", func() { - Expect(set.Name).To(HavePrefix(deployment.Name + "-")) + It("should create a firewall", func() { + fw = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 1, &v2.FirewallList{}, func(l *v2.FirewallList) []*v2.Firewall { + return l.GetItems() + }, 15*time.Second) }) - It("should be in the same namespace as the deployment", func() { - Expect(set.Namespace).To(Equal(deployment.Namespace)) + It("should create a firewall monitor", func() { + mon = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 1, &v2.FirewallMonitorList{}, func(l *v2.FirewallMonitorList) []*v2.FirewallMonitor { + return l.GetItems() + }, 15*time.Second) }) - It("should take the same replicas as defined by the deployement", func() { - Expect(set.Spec.Replicas).To(Equal(1)) + It("should allow an update of the firewall monitor", func() { + // simulating a firewall-controller updating the resource + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(mon), mon)).To(Succeed()) // refetch + mon.ControllerStatus = &v2.ControllerStatus{ + Updated: metav1.NewTime(time.Now()), + Distance: v2.FirewallShortestDistance, + DistanceSupported: true, + } + Expect(k8sClient.Update(ctx, mon)).To(Succeed()) }) - It("should inherit the spec from the deployment", func() { - wantSpec := deployment.Spec.Template.Spec.DeepCopy() - wantSpec.Size = "n2-medium-x86" // this is the change that triggered the rolling update - Expect(&set.Spec.Template.Spec).To(BeComparableTo(wantSpec)) - }) + Context("the firewall resource", func() { + It("should be named after the namespace (it's the shoot name in the end)", func() { + Expect(fw.Name).To(HavePrefix(namespaceName + "-firewall-")) + }) - It("should start with a higher distance", func() { - Expect(set.Spec.Distance).To(Equal(v2.FirewallRollingUpdateSetDistance)) - }) + It("should be in the same namespace as the set", func() { + Expect(fw.Namespace).To(Equal(set.Namespace)) + }) - It("should have the deployment as an owner", func() { - Expect(set.ObjectMeta.OwnerReferences).To(HaveLen(1)) - Expect(set.ObjectMeta.OwnerReferences[0].Name).To(Equal(deployment.Name)) - }) + It("should inherit the spec from the set", func() { + wantSpec := set.Spec.Template.Spec.DeepCopy() + Expect(&fw.Spec).To(BeComparableTo(wantSpec)) + }) - It("should populate the status", func() { - var set = set.DeepCopy() - Eventually(func() int { - Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(set), set)).To(Succeed()) - return set.Status.ProgressingReplicas - }, 15*time.Second, interval).Should(Equal(1), "reach progressing replicas") + It("should have the set as an owner", func() { + Expect(fw.ObjectMeta.OwnerReferences).To(HaveLen(1)) + Expect(fw.ObjectMeta.OwnerReferences[0].Name).To(Equal(set.Name)) + }) - Expect(set.Status.TargetReplicas).To(Equal(1)) - Expect(set.Status.ReadyReplicas).To(Equal(0)) - Expect(set.Status.UnhealthyReplicas).To(Equal(0)) - Expect(set.Status.ObservedRevision).To(Equal(1)) - }) - }) + It("should have the created condition true", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallCreated, v2.ConditionTrue, 15*time.Second) - Context("the firewall deployment resource", func() { - It("should have the rbac condition true", func() { - cond := testcommon.WaitForCondition(k8sClient, ctx, deployment.DeepCopy(), func(fd *v2.FirewallDeployment) v2.Conditions { - return fd.Status.Conditions - }, v2.FirewallDeplomentRBACProvisioned, v2.ConditionTrue, 5*time.Second) + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Equal("Created")) + Expect(cond.Message).To(Equal(fmt.Sprintf("Firewall %q created successfully.", *firewall1.Allocation.Name))) + }) - Expect(cond.LastTransitionTime).NotTo(BeZero()) - Expect(cond.LastUpdateTime).NotTo(BeZero()) - Expect(cond.Reason).To(Equal("Provisioned")) - Expect(cond.Message).To(Equal("RBAC provisioned successfully.")) - }) + It("should populate the machine status", func() { + var status *v2.MachineStatus + var fw = fw.DeepCopy() + Eventually(func() *v2.MachineStatus { + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(fw), fw)).To(Succeed()) + status = fw.Status.MachineStatus + return status + }, 5*time.Second, interval).Should(Not(BeNil())) + + Expect(status.MachineID).To(Equal(*firewall1.ID)) + Expect(status.CrashLoop).To(Equal(false)) + Expect(status.Liveliness).To(Equal("Alive")) + Expect(status.LastEvent).NotTo(BeNil()) + Expect(status.LastEvent.Event).To(Equal("Phoned Home")) + Expect(status.LastEvent.Message).To(Equal("phoning home")) + }) - It("should have the available condition false", func() { - cond := testcommon.WaitForCondition(k8sClient, ctx, deployment.DeepCopy(), func(fd *v2.FirewallDeployment) v2.Conditions { - return fd.Status.Conditions - }, v2.FirewallDeplomentAvailable, v2.ConditionFalse, 5*time.Second) + It("should have the ready condition true", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallReady, v2.ConditionTrue, 15*time.Second) - Expect(cond.LastTransitionTime).NotTo(BeZero()) - Expect(cond.LastUpdateTime).NotTo(BeZero()) - Expect(cond.Reason).To(Equal("MinimumReplicasUnavailable")) - Expect(cond.Message).To(Equal("Deployment does not have minimum availability.")) - }) + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Equal("Ready")) + Expect(cond.Message).To(Equal(fmt.Sprintf("Firewall %q is phoning home and alive.", *firewall1.Allocation.Name))) + }) - It("should have the progress condition true", func() { - cond := testcommon.WaitForCondition(k8sClient, ctx, deployment.DeepCopy(), func(fd *v2.FirewallDeployment) v2.Conditions { - return fd.Status.Conditions - }, v2.FirewallDeplomentProgressing, v2.ConditionTrue, 15*time.Second) + It("should have the monitor condition true", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallMonitorDeployed, v2.ConditionTrue, 5*time.Second) - Expect(cond.LastTransitionTime).NotTo(BeZero()) - Expect(cond.LastUpdateTime).NotTo(BeZero()) - Expect(cond.Reason).To(Or(Equal("NewFirewallSetAvailable"), Equal("FirewallSetUpdated"))) - Expect(cond.Message).To(Or( - Equal(fmt.Sprintf("FirewallSet %q has successfully progressed.", set.Name)), - Equal(fmt.Sprintf("Updated firewall set %q.", set.Name)), - )) - }) + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Equal("Deployed")) + Expect(cond.Message).To(Equal("Successfully deployed firewall-monitor.")) + }) - It("should populate the status", func() { - var deploy = deployment.DeepCopy() - Eventually(func() int { - Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(deploy), deploy)).To(Succeed()) - return deploy.Status.ProgressingReplicas - }, 15*time.Second, interval).Should(Equal(1), "reach progressing replicas") + It("should have the firewall-controller connected condition true", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallControllerConnected, v2.ConditionTrue, 15*time.Second) - Expect(deploy.Status.TargetReplicas).To(Equal(1)) - Expect(deploy.Status.ReadyReplicas).To(Equal(0)) - Expect(deploy.Status.UnhealthyReplicas).To(Equal(0)) - Expect(deploy.Status.ObservedRevision).To(Equal(1)) - }) + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Equal("Connected")) + Expect(cond.Message).To(Equal(fmt.Sprintf("Controller reconciled firewall at %s.", mon.ControllerStatus.Updated.Time.String()))) + }) - It("should not be possible to update deployment strategy while deployment has not converged", func() { - var deploy = deploy.DeepCopy() - Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(deploy), deploy)).To(Succeed()) - deploy.Spec.Strategy = v2.StrategyRecreate - err := k8sClient.Update(ctx, deploy) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring(`Invalid value: "Recreate": strategy can not be updated until target replicas have been reached (i.e. deployment has converged)`)) + It("should have configured the distance", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallDistanceConfigured, v2.ConditionTrue, 15*time.Second) + + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Equal("Configured")) + Expect(cond.Message).To(Equal(fmt.Sprintf("Controller has configured the specified distance %d.", v2.FirewallShortestDistance))) + }) }) - }) - var ( - readyFirewall = firewall2("Phoned Home", "is phoning home") - ) + Context("the firewall set resource", func() { + It("should be named after the deployment", func() { + Expect(set.Name).To(HavePrefix(deployment().Name + "-")) + }) - When("the firewall gets ready and the firewall-controller connects", Ordered, func() { - It("should allow an update of the firewall monitor", func() { - swapMetalClient(&metalclient.MetalMockFns{ - Machine: func(m *mock.Mock) { - m.On("FreeMachine", mock.Anything, nil).Return(&machine.FreeMachineOK{Payload: &models.V1MachineResponse{ID: firewall1.ID}}, nil).Maybe() - m.On("UpdateMachine", mock.Anything, nil).Return(&machine.UpdateMachineOK{Payload: &models.V1MachineResponse{}}, nil).Maybe() - }, - Firewall: func(m *mock.Mock) { - m.On("FindFirewalls", mock.Anything, nil).Return(&metalfirewall.FindFirewallsOK{Payload: []*models.V1FirewallResponse{readyFirewall}}, nil).Maybe() - }, - Network: func(m *mock.Mock) { - m.On("FindNetwork", mock.Anything, nil).Return(&network.FindNetworkOK{Payload: network1}, nil).Maybe() - }, + It("should be in the same namespace as the deployment", func() { + Expect(set.Namespace).To(Equal(deployment().Namespace)) }) - Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(mon), mon)).To(Succeed()) // refetch - // simulating a firewall-controller updating the resource - mon.ControllerStatus = &v2.ControllerStatus{ - Updated: metav1.NewTime(time.Now()), - Distance: v2.FirewallRollingUpdateSetDistance, - DistanceSupported: true, - } - Expect(k8sClient.Update(ctx, mon)).To(Succeed()) + It("should take the same replicas as defined by the deployment", func() { + Expect(set.Spec.Replicas).To(Equal(1)) + }) + + It("should inherit the spec from the deployment", func() { + deploy := &v2.FirewallDeployment{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(deployment()), deploy)).To(Succeed()) + Expect(set.Spec.Template.Spec).To(BeComparableTo(deploy.Spec.Template.Spec)) + }) + + It("should have the deployment as an owner", func() { + Expect(set.ObjectMeta.OwnerReferences).To(HaveLen(1)) + Expect(set.ObjectMeta.OwnerReferences[0].Name).To(Equal(deployment().Name)) + }) + + It("should populate the status", func() { + var set = set.DeepCopy() + Eventually(func() int { + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(set), set)).To(Succeed()) + return set.Status.ReadyReplicas + }, 15*time.Second, interval).Should(Equal(1), "reach ready replicas") + + Expect(set.Status.TargetReplicas).To(Equal(1)) + Expect(set.Status.ProgressingReplicas).To(Equal(0)) + Expect(set.Status.UnhealthyReplicas).To(Equal(0)) + Expect(set.Status.ObservedRevision).To(Equal(0)) // this is the first revision + }) }) - It("the firewall should reflect the distance during a rolling update", func() { - cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { - return fd.Status.Conditions - }, v2.FirewallDistanceConfigured, v2.ConditionTrue, 15*time.Second) + Context("the firewall deployment resource", func() { + It("should have the update strategy recreate", func() { + deploy := &v2.FirewallDeployment{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(deployment()), deploy)).To(Succeed()) + Expect(deploy.Spec.Strategy).To(Equal(v2.StrategyRecreate)) + }) - Expect(cond.LastTransitionTime).NotTo(BeZero()) - Expect(cond.LastUpdateTime).NotTo(BeZero()) - Expect(cond.Reason).To(Equal("Configured")) - Expect(cond.Message).To(Equal(fmt.Sprintf("Controller has configured the specified distance %d.", v2.FirewallRollingUpdateSetDistance))) + It("should have the rbac condition true", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, deployment(), func(fd *v2.FirewallDeployment) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallDeplomentRBACProvisioned, v2.ConditionTrue, 5*time.Second) + + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Equal("Provisioned")) + Expect(cond.Message).To(Equal("RBAC provisioned successfully.")) + }) + + It("should have the available condition true", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, deployment(), func(fd *v2.FirewallDeployment) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallDeplomentAvailable, v2.ConditionTrue, 5*time.Second) + + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Equal("MinimumReplicasAvailable")) + Expect(cond.Message).To(Equal("Deployment has minimum availability.")) + }) + + It("should have the progress condition true", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, deployment(), func(fd *v2.FirewallDeployment) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallDeplomentProgressing, v2.ConditionTrue, 15*time.Second) + + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Or(Equal("NewFirewallSetAvailable"), Equal("FirewallSetUpdated"))) + Expect(cond.Message).To(Or( + Equal(fmt.Sprintf("FirewallSet %q has successfully progressed.", set.Name)), + Equal(fmt.Sprintf("Updated firewall set %q.", set.Name)), + )) + }) + + It("should populate the status", func() { + deploy := &v2.FirewallDeployment{} + Eventually(func() int { + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(deployment()), deploy)).To(Succeed()) + return deploy.Status.ReadyReplicas + }, 15*time.Second, interval).Should(Equal(1), "reach ready replicas") + + Expect(deploy.Status.TargetReplicas).To(Equal(1)) + Expect(deploy.Status.ProgressingReplicas).To(Equal(0)) + Expect(deploy.Status.UnhealthyReplicas).To(Equal(0)) + Expect(deploy.Status.ObservedRevision).To(Equal(0)) // this is the first revision + }) }) + + // TODO: verify monitor resource }) - Context("the old generation disappears", func() { - It("should delete the firewall set", func() { - set = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 1, &v2.FirewallSetList{}, func(l *v2.FirewallSetList) []*v2.FirewallSet { + When("a significant change occurs", Ordered, func() { + var ( + readyFirewall = firewall2("Phoned Home", "is phoning home") + + oldFw *v2.Firewall + oldSet *v2.FirewallSet + oldMon *v2.FirewallMonitor + ) + + It("should find a single firewall set before the update", func() { + oldSet = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 1, &v2.FirewallSetList{}, func(l *v2.FirewallSetList) []*v2.FirewallSet { return l.GetItems() - }, 15*time.Second) - Expect(set.Status.ObservedRevision).To(Equal(1)) + }, 5*time.Second) }) - It("should delete the firewall", func() { - fw = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 1, &v2.FirewallList{}, func(l *v2.FirewallList) []*v2.Firewall { + It("should find a single firewall before the update", func() { + oldFw = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 1, &v2.FirewallList{}, func(l *v2.FirewallList) []*v2.Firewall { return l.GetItems() - }, 3*time.Second) - Expect(fw.Status.MachineStatus.MachineID).To(Equal(*readyFirewall.ID)) + }, 5*time.Second) }) - It("should delete firewall monitor", func() { - mon = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 1, &v2.FirewallMonitorList{}, func(l *v2.FirewallMonitorList) []*v2.FirewallMonitor { + It("should find a single firewall monitor before the update", func() { + oldMon = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 1, &v2.FirewallMonitorList{}, func(l *v2.FirewallMonitorList) []*v2.FirewallMonitor { return l.GetItems() }, 5*time.Second) - Expect(mon.MachineStatus.MachineID).To(Equal(*readyFirewall.ID)) }) - It("should populate the controller status field in the firewall resource", func() { - var fw = fw.DeepCopy() - Eventually(func() *v2.ControllerConnection { - Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(fw), fw)).To(Succeed()) - return fw.Status.ControllerStatus - }, 15*time.Second, interval).Should(Not(BeNil()), "controller connection was not synced to firewall resource") - }) + Context("the spec is updated", Ordered, func() { + It("the update works", func() { + deploy := deployment() - It("the firewall set should be updated to shortest distance as the update has succeeded", func() { - Eventually(func() v2.FirewallDistance { - Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(set), set)).To(Succeed()) - return set.Spec.Distance - }).Within(5 * time.Second).ProbeEvery(200 * time.Millisecond).Should(Equal(v2.FirewallShortestDistance)) - Expect(set.Spec.Distance).To(Equal(v2.FirewallShortestDistance)) - }) - }) - }) + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(deployment()), deploy)).To(Succeed()) + + swapMetalClient(&metalclient.MetalMockFns{ + Firewall: func(m *mock.Mock) { + m.On("FindFirewalls", mock.Anything, nil).Return(&metalfirewall.FindFirewallsOK{Payload: []*models.V1FirewallResponse{readyFirewall}}, nil).Maybe() + }, + Network: func(m *mock.Mock) { + m.On("FindNetwork", mock.Anything, nil).Return(&network.FindNetworkOK{Payload: network1}, nil).Maybe() + }, + Machine: func(m *mock.Mock) { + m.On("FreeMachine", mock.Anything, nil).Return(nil, &machine.FreeMachineDefault{Payload: httperrors.Conflict(fmt.Errorf("deletion blocked"))}).Maybe() + m.On("UpdateMachine", mock.Anything, nil).Return(&machine.UpdateMachineOK{Payload: &models.V1MachineResponse{}}, nil).Maybe() + }, + }) + + deploy.Spec.Template.Spec.Networks = []string{"internet", "mpls"} + + Expect(k8sClient.Update(ctx, deploy)).To(Succeed()) + }) - Describe("the deletion flow", Ordered, func() { - When("deleting the firewall deployment", func() { - It("the deletion finishes", func() { - Expect(k8sClient.Delete(ctx, deployment)).To(Succeed()) + It("a new generation appears with 0 replicas", func() { + set := &v2.FirewallSet{} + Eventually(func() int { + set = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 2, &v2.FirewallSetList{}, func(l *v2.FirewallSetList) []*v2.FirewallSet { + return l.GetItems() + }, 15*time.Second) + return set.Spec.Replicas + }, 15*time.Second, interval).Should(Equal(0)) + Eventually(func() int { + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(set), set)).To(Succeed()) + return set.Status.ObservedRevision // revision is set after first status update, before that it's 0 + }, 3*time.Second, interval).Should(Equal(1)) + }) + + Context("the old generation disappears", Ordered, func() { + It("should delete the firewall set", func() { + swapMetalClient(&metalclient.MetalMockFns{ + Firewall: func(m *mock.Mock) { + m.On("AllocateFirewall", mock.Anything, nil).Return(&metalfirewall.AllocateFirewallOK{Payload: readyFirewall}, nil).Maybe() + m.On("FindFirewalls", mock.Anything, nil).Return(&metalfirewall.FindFirewallsOK{Payload: []*models.V1FirewallResponse{readyFirewall}}, nil).Maybe() + }, + Network: func(m *mock.Mock) { + m.On("FindNetwork", mock.Anything, nil).Return(&network.FindNetworkOK{Payload: network1}, nil).Maybe() + }, + Machine: func(m *mock.Mock) { + m.On("FreeMachine", mock.Anything, nil).Return(&machine.FreeMachineOK{Payload: &models.V1MachineResponse{ID: firewall1.ID}}, nil).Maybe() + m.On("UpdateMachine", mock.Anything, nil).Return(&machine.UpdateMachineOK{Payload: &models.V1MachineResponse{}}, nil).Maybe() + }, + }) + + Eventually(func() bool { + err := k8sClient.Get(ctx, client.ObjectKeyFromObject(oldSet), oldSet) + return apierrors.IsNotFound(err) + }, 15*time.Second, interval).Should(BeTrue()) + }) + + It("should delete the firewall", func() { + Eventually(func() bool { + err := k8sClient.Get(ctx, client.ObjectKeyFromObject(oldFw), oldFw) + return apierrors.IsNotFound(err) + }, 15*time.Second, interval).Should(BeTrue()) + }) + + It("should delete firewall monitor", func() { + Eventually(func() bool { + err := k8sClient.Get(ctx, client.ObjectKeyFromObject(oldMon), oldMon) + return apierrors.IsNotFound(err) + }, 15*time.Second, interval).Should(BeTrue()) + }) + }) + + var ( + newFw *v2.Firewall + newSet *v2.FirewallSet + newMon *v2.FirewallMonitor + ) + + Context("a new generation is scaled up", func() { + It("should scale up the new set", func() { + Eventually(func() int { + newSet = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 1, &v2.FirewallSetList{}, func(l *v2.FirewallSetList) []*v2.FirewallSet { + return l.GetItems() + }, 15*time.Second) + return newSet.Spec.Replicas + }, 15*time.Second, interval).Should(Equal(1)) + }) + + It("should create a new firewall", func() { + newFw = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 1, &v2.FirewallList{}, func(l *v2.FirewallList) []*v2.Firewall { + return l.GetItems() + }, 15*time.Second) + }) + + It("should create a firewall monitor", func() { + newMon = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 1, &v2.FirewallMonitorList{}, func(l *v2.FirewallMonitorList) []*v2.FirewallMonitor { + return l.GetItems() + }, 15*time.Second) + }) + }) + + When("the firewall-controller connects", Ordered, func() { + It("should allow an update of the firewall monitor", func() { + swapMetalClient(&metalclient.MetalMockFns{ + Machine: func(m *mock.Mock) { + m.On("UpdateMachine", mock.Anything, nil).Return(&machine.UpdateMachineOK{Payload: &models.V1MachineResponse{}}, nil).Maybe() + }, + Firewall: func(m *mock.Mock) { + m.On("FindFirewalls", mock.Anything, nil).Return(&metalfirewall.FindFirewallsOK{Payload: []*models.V1FirewallResponse{readyFirewall}}, nil).Maybe() + }, + Network: func(m *mock.Mock) { + m.On("FindNetwork", mock.Anything, nil).Return(&network.FindNetworkOK{Payload: network1}, nil).Maybe() + }, + }) + + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(newMon), newMon)).To(Succeed()) // refetch + // simulating a firewall-controller updating the resource + newMon.ControllerStatus = &v2.ControllerStatus{ + Updated: metav1.NewTime(time.Now()), + Distance: v2.FirewallShortestDistance, + DistanceSupported: true, + } + Expect(k8sClient.Update(ctx, newMon)).To(Succeed()) + }) + }) + + Context("the new firewall resource", func() { + It("should be named after the namespace (it's the shoot name in the end)", func() { + Expect(newFw.Name).To(HavePrefix(namespaceName + "-firewall-")) + }) + + It("should be in the same namespace as the set", func() { + Expect(newFw.Namespace).To(Equal(newSet.Namespace)) + }) + + It("should inherit the spec from the set", func() { + wantSpec := newSet.Spec.Template.Spec.DeepCopy() + Expect(&newFw.Spec).To(BeComparableTo(wantSpec)) + }) + + It("should have the set as an owner", func() { + Expect(newFw.ObjectMeta.OwnerReferences).To(HaveLen(1)) + Expect(newFw.ObjectMeta.OwnerReferences[0].Name).To(Equal(newSet.Name)) + }) + + It("should have the created condition true", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, newFw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallCreated, v2.ConditionTrue, 15*time.Second) + + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Equal("Created")) + Expect(cond.Message).To(Equal(fmt.Sprintf("Firewall %q created successfully.", *readyFirewall.Allocation.Name))) + }) + + It("should populate the machine status", func() { + var status *v2.MachineStatus + var fw = newFw.DeepCopy() + Eventually(func() *v2.MachineStatus { + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(fw), fw)).To(Succeed()) + status = fw.Status.MachineStatus + return status + }, 5*time.Second, interval).Should(Not(BeNil())) + + Expect(status.MachineID).To(Equal(*readyFirewall.ID)) + Expect(status.CrashLoop).To(Equal(false)) + Expect(status.Liveliness).To(Equal("Alive")) + Expect(status.LastEvent).NotTo(BeNil()) + Expect(status.LastEvent.Event).To(Equal("Phoned Home")) + Expect(status.LastEvent.Message).To(Equal("is phoning home")) + }) + + It("should have the ready condition true", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, newFw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallReady, v2.ConditionTrue, 15*time.Second) + + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Equal("Ready")) + Expect(cond.Message).To(Equal(fmt.Sprintf("Firewall %q is phoning home and alive.", *readyFirewall.Allocation.Name))) + }) + + It("should have the monitor condition true", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, newFw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallMonitorDeployed, v2.ConditionTrue, 5*time.Second) + + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Equal("Deployed")) + Expect(cond.Message).To(Equal("Successfully deployed firewall-monitor.")) + }) + + It("should have the firewall-controller connected condition true", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, newFw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallControllerConnected, v2.ConditionTrue, 15*time.Second) + + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Equal("Connected")) + Expect(cond.Message).To(Equal(fmt.Sprintf("Controller reconciled firewall at %s.", newMon.ControllerStatus.Updated.Time.String()))) + }) + + It("should have firewall networks populated", func() { + var nws []v2.FirewallNetwork + var fw = newFw.DeepCopy() + Eventually(func() []v2.FirewallNetwork { + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(fw), fw)).To(Succeed()) + nws = fw.Status.FirewallNetworks + return nws + }, 5*time.Second, interval).Should(HaveLen(1)) + + Expect(nws).To(BeComparableTo([]v2.FirewallNetwork{ + { + ASN: readyFirewall.Allocation.Networks[0].Asn, + DestinationPrefixes: readyFirewall.Allocation.Networks[0].Destinationprefixes, + IPs: readyFirewall.Allocation.Networks[0].Ips, + Nat: readyFirewall.Allocation.Networks[0].Nat, + NetworkID: readyFirewall.Allocation.Networks[0].Networkid, + NetworkType: readyFirewall.Allocation.Networks[0].Networktype, + Prefixes: network1.Prefixes, + Vrf: readyFirewall.Allocation.Networks[0].Vrf, + }, + })) + }) + + It("should have shoot access populated", func() { + var access *v2.ShootAccess + var fw = newFw.DeepCopy() + Eventually(func() *v2.ShootAccess { + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(fw), fw)).To(Succeed()) + access = fw.Status.ShootAccess + return access + }, 5*time.Second, interval).Should(Not(BeNil())) + + Expect(access).To(BeComparableTo(&v2.ShootAccess{ + GenericKubeconfigSecretName: "kubeconfig-secret-name", + TokenSecretName: "token", + Namespace: namespaceName, + APIServerURL: apiHost, + SSHKeySecretName: sshSecret.Name, + })) + }) + + It("should have the shortest distance", func() { + Expect(newFw.Distance).To(Equal(v2.FirewallShortestDistance)) + }) + }) + + Context("the new firewall set resource", func() { + It("should be named after the deployment", func() { + Expect(newSet.Name).To(HavePrefix(deployment().Name + "-")) + }) + + It("should be in the same namespace as the deployment", func() { + Expect(newSet.Namespace).To(Equal(deployment().Namespace)) + }) + + It("should inherit the spec from the deployment", func() { + deploy := &v2.FirewallDeployment{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(deployment()), deploy)).To(Succeed()) + wantSpec := deploy.Spec.Template.Spec.DeepCopy() + wantSpec.Networks = []string{"internet", "mpls"} // this is the change that triggered the recreate update + Expect(&newSet.Spec.Template.Spec).To(BeComparableTo(wantSpec)) + }) + + It("should have the shortest distance", func() { + Expect(newSet.Spec.Distance).To(Equal(v2.FirewallShortestDistance)) + }) + + It("should have the deployment as an owner", func() { + Expect(newSet.ObjectMeta.OwnerReferences).To(HaveLen(1)) + Expect(newSet.ObjectMeta.OwnerReferences[0].Name).To(Equal(deployment().Name)) + }) + + It("should populate the status", func() { + var set = newSet.DeepCopy() + Eventually(func() int { + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(set), set)).To(Succeed()) + return set.Status.ReadyReplicas + }, 15*time.Second, interval).Should(Equal(1), "reach ready replicas") + + Expect(set.Status.TargetReplicas).To(Equal(1)) + Expect(set.Status.ProgressingReplicas).To(Equal(0)) + Expect(set.Status.UnhealthyReplicas).To(Equal(0)) + Expect(set.Status.ObservedRevision).To(Equal(1)) + }) + }) + + Context("the firewall deployment resource", func() { + It("should have the rbac condition true", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, deployment(), func(fd *v2.FirewallDeployment) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallDeplomentRBACProvisioned, v2.ConditionTrue, 5*time.Second) + + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Equal("Provisioned")) + Expect(cond.Message).To(Equal("RBAC provisioned successfully.")) + }) + + It("should have the available condition true", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, deployment(), func(fd *v2.FirewallDeployment) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallDeplomentAvailable, v2.ConditionTrue, 5*time.Second) + + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Equal("MinimumReplicasAvailable")) + Expect(cond.Message).To(Equal("Deployment has minimum availability.")) + }) + + It("should have the progress condition true", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, deployment(), func(fd *v2.FirewallDeployment) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallDeplomentProgressing, v2.ConditionTrue, 15*time.Second) + + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Or(Equal("NewFirewallSetAvailable"), Equal("FirewallSetUpdated"))) + Expect(cond.Message).To(Or( + Equal(fmt.Sprintf("FirewallSet %q has successfully progressed.", newSet.Name)), + Equal(fmt.Sprintf("Updated firewall set %q.", newSet.Name)), + )) + }) + + It("should populate the status", func() { + var deploy = deployment() + Eventually(func() int { + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(deploy), deploy)).To(Succeed()) + return deploy.Status.ReadyReplicas + }, 15*time.Second, interval).Should(Equal(1), "reach 1 ready replicas") + Eventually(func() int { + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(deploy), deploy)).To(Succeed()) + return deploy.Status.ProgressingReplicas + }, 5*time.Second, interval).Should(Equal(0), "reach 0 progressing replicas") + + Expect(deploy.Status.TargetReplicas).To(Equal(1)) + Expect(deploy.Status.UnhealthyReplicas).To(Equal(0)) + Expect(deploy.Status.ObservedRevision).To(Equal(1)) + }) + }) }) }) - Context("all resources are cleaned up", func() { - It("should delete the firewall set", func() { - _ = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 0, &v2.FirewallSetList{}, func(l *v2.FirewallSetList) []*v2.FirewallSet { - return l.GetItems() - }, 10*time.Second) - }) + Describe("the deletion flow", Ordered, func() { + When("deleting the firewall deployment", func() { + It("the deletion finishes", func() { + swapMetalClient(&metalclient.MetalMockFns{ + Firewall: func(m *mock.Mock) { + m.On("AllocateFirewall", mock.Anything, nil).Return(&metalfirewall.AllocateFirewallOK{Payload: firewall1}, nil).Maybe() + m.On("FindFirewalls", mock.Anything, nil).Return(&metalfirewall.FindFirewallsOK{Payload: []*models.V1FirewallResponse{firewall1}}, nil).Maybe() + }, + Network: func(m *mock.Mock) { + m.On("FindNetwork", mock.Anything, nil).Return(&network.FindNetworkOK{Payload: network1}, nil).Maybe() + }, + Machine: func(m *mock.Mock) { + m.On("FreeMachine", mock.Anything, nil).Return(&machine.FreeMachineOK{Payload: &models.V1MachineResponse{ID: firewall1.ID}}, nil).Maybe() + m.On("UpdateMachine", mock.Anything, nil).Return(&machine.UpdateMachineOK{Payload: &models.V1MachineResponse{}}, nil).Maybe() + }, + }) - It("should delete the firewall", func() { - _ = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 0, &v2.FirewallList{}, func(l *v2.FirewallList) []*v2.Firewall { - return l.GetItems() - }, 10*time.Second) + Expect(k8sClient.Delete(ctx, deployment())).To(Succeed()) + }) }) - It("should delete firewall monitor", func() { - _ = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 0, &v2.FirewallMonitorList{}, func(l *v2.FirewallMonitorList) []*v2.FirewallMonitor { - return l.GetItems() - }, 10*time.Second) + Context("all resources are cleaned up", func() { + It("should delete the firewall set", func() { + _ = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 0, &v2.FirewallSetList{}, func(l *v2.FirewallSetList) []*v2.FirewallSet { + return l.GetItems() + }, 10*time.Second) + }) + + It("should delete the firewall", func() { + _ = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 0, &v2.FirewallList{}, func(l *v2.FirewallList) []*v2.Firewall { + return l.GetItems() + }, 10*time.Second) + }) + + It("should delete firewall monitor", func() { + _ = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 0, &v2.FirewallMonitorList{}, func(l *v2.FirewallMonitorList) []*v2.FirewallMonitor { + return l.GetItems() + }, 10*time.Second) + }) }) }) }) -}) -var _ = Context("migration path", Ordered, func() { - var ( - fw = &v2.Firewall{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - Namespace: namespaceName, - Labels: map[string]string{ - "purpose": "shoot-firewall", + Context("migration path", Ordered, func() { + var ( + fw = &v2.Firewall{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: namespaceName, + Labels: map[string]string{ + "purpose": "shoot-firewall", + }, + Annotations: map[string]string{ + v2.FirewallNoControllerConnectionAnnotation: "true", + }, }, - Annotations: map[string]string{ - v2.FirewallNoControllerConnectionAnnotation: "true", + Spec: v2.FirewallSpec{ + Size: "n1-medium-x86", + Project: "project-a", + Partition: "partition-a", + Image: "firewall-ubuntu-2.0", + Networks: []string{"internet"}, + ControllerURL: "http://controller.tar.gz", + ControllerVersion: "v2.0.0", + NftablesExporterURL: "http://exporter.tar.gz", + NftablesExporterVersion: "v1.0.0", }, - }, - Spec: v2.FirewallSpec{ - Size: "n1-medium-x86", - Project: "project-a", - Partition: "partition-a", - Image: "firewall-ubuntu-2.0", - Networks: []string{"internet"}, - ControllerURL: "http://controller.tar.gz", - ControllerVersion: "v2.0.0", - NftablesExporterURL: "http://exporter.tar.gz", - NftablesExporterVersion: "v1.0.0", - }, - } + } - deployment = &v2.FirewallDeployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - Namespace: namespaceName, - }, - Spec: v2.FirewallDeploymentSpec{ - Template: v2.FirewallTemplateSpec{ + deployment = func() *v2.FirewallDeployment { + return &v2.FirewallDeployment{ ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "purpose": "shoot-firewall", - }, + Name: "test", + Namespace: namespaceName, }, - Spec: v2.FirewallSpec{ - Size: "n1-medium-x86", - Project: "project-a", - Partition: "partition-a", - Image: "firewall-ubuntu-2.0", - Networks: []string{"internet"}, - ControllerURL: "http://controller.tar.gz", - ControllerVersion: "v2.0.0", - NftablesExporterURL: "http://exporter.tar.gz", - NftablesExporterVersion: "v1.0.0", + Spec: v2.FirewallDeploymentSpec{ + Replicas: 1, + Template: v2.FirewallTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "purpose": "shoot-firewall", + }, + }, + Spec: v2.FirewallSpec{ + Size: "n1-medium-x86", + Project: "project-a", + Partition: "partition-a", + Image: "firewall-ubuntu-2.0", + Networks: []string{"internet"}, + ControllerURL: "http://controller.tar.gz", + ControllerVersion: "v2.0.0", + NftablesExporterURL: "http://exporter.tar.gz", + NftablesExporterVersion: "v1.0.0", + }, + }, }, - }, - }, - } - ) - - BeforeAll(func() { - Expect(client.IgnoreAlreadyExists(k8sClient.Create(ctx, namespace.DeepCopy()))).To(Succeed()) - Expect(client.IgnoreAlreadyExists(k8sClient.Create(ctx, fakeTokenSecretSeed.DeepCopy()))).To(Succeed()) - Expect(client.IgnoreAlreadyExists(k8sClient.Create(ctx, fakeTokenSecretShoot.DeepCopy()))).To(Succeed()) - Expect(client.IgnoreAlreadyExists(k8sClient.Create(ctx, sshSecret.DeepCopy()))).To(Succeed()) - Expect(client.IgnoreAlreadyExists(k8sClient.Create(ctx, genericKubeconfigSecret(apiCA, apiHost, apiCert, apiKey)))).To(Succeed()) - Expect(client.IgnoreAlreadyExists(k8sClient.Create(ctx, shootTokenSecret.DeepCopy()))).To(Succeed()) - DeferCleanup(func() { - swapMetalClient(&metalclient.MetalMockFns{ - Firewall: func(m *mock.Mock) { - m.On("FindFirewalls", mock.Anything, nil).Return(&metalfirewall.FindFirewallsOK{Payload: []*models.V1FirewallResponse{firewall1}}, nil).Maybe() - }, - Network: func(m *mock.Mock) { - m.On("FindNetwork", mock.Anything, nil).Return(&network.FindNetworkOK{Payload: network1}, nil).Maybe() - }, - Machine: func(m *mock.Mock) { - m.On("FreeMachine", mock.Anything, nil).Return(&machine.FreeMachineOK{Payload: &models.V1MachineResponse{ID: firewall1.ID}}, nil).Maybe() - m.On("UpdateMachine", mock.Anything, nil).Return(&machine.UpdateMachineOK{Payload: &models.V1MachineResponse{}}, nil).Maybe() - }, - }) - Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, deployment.DeepCopy()))).To(Succeed()) - _ = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 0, &v2.FirewallDeploymentList{}, func(l *v2.FirewallDeploymentList) []*v2.FirewallDeployment { - return l.GetItems() - }, 10*time.Second) - }) - }) - - Describe("the migration starts", Ordered, func() { - swapMetalClient(&metalclient.MetalMockFns{ - Firewall: func(m *mock.Mock) { - m.On("FindFirewalls", mock.Anything, nil).Return(&metalfirewall.FindFirewallsOK{Payload: []*models.V1FirewallResponse{firewall1}}, nil).Maybe() - }, - Network: func(m *mock.Mock) { - m.On("FindNetwork", mock.Anything, nil).Return(&network.FindNetworkOK{Payload: network1}, nil).Maybe() - }, - Machine: func(m *mock.Mock) { - m.On("UpdateMachine", mock.Anything, nil).Return(&machine.UpdateMachineOK{Payload: &models.V1MachineResponse{}}, nil).Maybe() - }, - }) + } + } + ) When("creating a firewall resource (for an existing firewall)", Ordered, func() { It("the creation works", func() { + swapMetalClient(&metalclient.MetalMockFns{ + Firewall: func(m *mock.Mock) { + m.On("FindFirewalls", mock.Anything, nil).Return(&metalfirewall.FindFirewallsOK{Payload: []*models.V1FirewallResponse{firewall1}}, nil).Maybe() + }, + Network: func(m *mock.Mock) { + m.On("FindNetwork", mock.Anything, nil).Return(&network.FindNetworkOK{Payload: network1}, nil).Maybe() + }, + Machine: func(m *mock.Mock) { + m.On("UpdateMachine", mock.Anything, nil).Return(&machine.UpdateMachineOK{Payload: &models.V1MachineResponse{}}, nil).Maybe() + }, + }) + Expect(k8sClient.Create(ctx, fw)).To(Succeed()) }) @@ -936,7 +1615,7 @@ var _ = Context("migration path", Ordered, func() { When("creating a firewall deployment", Ordered, func() { It("the creation works", func() { - Expect(k8sClient.Create(ctx, deployment)).To(Succeed()) + Expect(k8sClient.Create(ctx, deployment())).To(Succeed()) }) It("should create a firewall set", func() { @@ -1034,7 +1713,7 @@ var _ = Context("migration path", Ordered, func() { Context("the firewall set resource", func() { It("should have the deployment as an owner", func() { Expect(set.ObjectMeta.OwnerReferences).To(HaveLen(1)) - Expect(set.ObjectMeta.OwnerReferences[0].Name).To(Equal(deployment.Name)) + Expect(set.ObjectMeta.OwnerReferences[0].Name).To(Equal(deployment().Name)) }) It("should populate the status", func() { @@ -1053,7 +1732,7 @@ var _ = Context("migration path", Ordered, func() { Context("the firewall deployment resource", func() { It("should have the rbac condition true", func() { - cond := testcommon.WaitForCondition(k8sClient, ctx, deployment.DeepCopy(), func(fd *v2.FirewallDeployment) v2.Conditions { + cond := testcommon.WaitForCondition(k8sClient, ctx, deployment(), func(fd *v2.FirewallDeployment) v2.Conditions { return fd.Status.Conditions }, v2.FirewallDeplomentRBACProvisioned, v2.ConditionTrue, 5*time.Second) @@ -1064,7 +1743,7 @@ var _ = Context("migration path", Ordered, func() { }) It("should have the available condition true", func() { - cond := testcommon.WaitForCondition(k8sClient, ctx, deployment.DeepCopy(), func(fd *v2.FirewallDeployment) v2.Conditions { + cond := testcommon.WaitForCondition(k8sClient, ctx, deployment(), func(fd *v2.FirewallDeployment) v2.Conditions { return fd.Status.Conditions }, v2.FirewallDeplomentAvailable, v2.ConditionTrue, 5*time.Second) @@ -1075,7 +1754,7 @@ var _ = Context("migration path", Ordered, func() { }) It("should have the progress condition true", func() { - cond := testcommon.WaitForCondition(k8sClient, ctx, deployment.DeepCopy(), func(fd *v2.FirewallDeployment) v2.Conditions { + cond := testcommon.WaitForCondition(k8sClient, ctx, deployment(), func(fd *v2.FirewallDeployment) v2.Conditions { return fd.Status.Conditions }, v2.FirewallDeplomentProgressing, v2.ConditionTrue, 15*time.Second) @@ -1089,9 +1768,9 @@ var _ = Context("migration path", Ordered, func() { }) It("should populate the status", func() { - var deploy = deployment.DeepCopy() + deploy := &v2.FirewallDeployment{} Eventually(func() int { - Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(deploy), deploy)).To(Succeed()) + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(deployment()), deploy)).To(Succeed()) return deploy.Status.ReadyReplicas }, 15*time.Second, interval).Should(Equal(1), "reach ready replicas") @@ -1101,5 +1780,47 @@ var _ = Context("migration path", Ordered, func() { Expect(deploy.Status.ObservedRevision).To(Equal(0)) // this is the first revision }) }) + + Describe("the deletion flow", Ordered, func() { + When("deleting the firewall deployment", func() { + It("the deletion finishes", func() { + swapMetalClient(&metalclient.MetalMockFns{ + Firewall: func(m *mock.Mock) { + m.On("AllocateFirewall", mock.Anything, nil).Return(&metalfirewall.AllocateFirewallOK{Payload: firewall1}, nil).Maybe() + m.On("FindFirewalls", mock.Anything, nil).Return(&metalfirewall.FindFirewallsOK{Payload: []*models.V1FirewallResponse{firewall1}}, nil).Maybe() + }, + Network: func(m *mock.Mock) { + m.On("FindNetwork", mock.Anything, nil).Return(&network.FindNetworkOK{Payload: network1}, nil).Maybe() + }, + Machine: func(m *mock.Mock) { + m.On("FreeMachine", mock.Anything, nil).Return(&machine.FreeMachineOK{Payload: &models.V1MachineResponse{ID: firewall1.ID}}, nil).Maybe() + m.On("UpdateMachine", mock.Anything, nil).Return(&machine.UpdateMachineOK{Payload: &models.V1MachineResponse{}}, nil).Maybe() + }, + }) + + Expect(k8sClient.Delete(ctx, deployment())).To(Succeed()) + }) + }) + + Context("all resources are cleaned up", func() { + It("should delete the firewall set", func() { + _ = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 0, &v2.FirewallSetList{}, func(l *v2.FirewallSetList) []*v2.FirewallSet { + return l.GetItems() + }, 10*time.Second) + }) + + It("should delete the firewall", func() { + _ = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 0, &v2.FirewallList{}, func(l *v2.FirewallList) []*v2.Firewall { + return l.GetItems() + }, 10*time.Second) + }) + + It("should delete firewall monitor", func() { + _ = testcommon.WaitForResourceAmount(k8sClient, ctx, namespaceName, 0, &v2.FirewallMonitorList{}, func(l *v2.FirewallMonitorList) []*v2.FirewallMonitor { + return l.GetItems() + }, 10*time.Second) + }) + }) + }) }) }) diff --git a/integration/suite_test.go b/integration/suite_test.go index 2a84d5a..83e3779 100644 --- a/integration/suite_test.go +++ b/integration/suite_test.go @@ -16,6 +16,7 @@ import ( "github.com/metal-stack/firewall-controller-manager/controllers/firewall" "github.com/metal-stack/firewall-controller-manager/controllers/monitor" "github.com/metal-stack/firewall-controller-manager/controllers/set" + metalclient "github.com/metal-stack/metal-go/test/client" "github.com/metal-stack/metal-lib/pkg/tag" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -99,6 +100,8 @@ var _ = BeforeSuite(func() { }) Expect(err).ToNot(HaveOccurred()) + metalClient, _ = metalclient.NewMetalMockClient(testingT, nil) + cc, err := controllerconfig.New(&controllerconfig.NewControllerConfig{ SeedClient: k8sClient, SeedConfig: cfg,