Bring back development in mini-lab. (#34)

Dockerfile.dev

@@ -0,0 +1,4 @@
+FROM alpine:3.17
+COPY bin/firewall-controller-manager /firewall-controller-manager
+USER 65534
+ENTRYPOINT ["/firewall-controller-manager"]

Makefile

@@ -38,9 +38,13 @@ manager: generate fmt vet
 						 -o bin/firewall-controller-manager main.go
 	strip bin/firewall-controller-manager
 
-# Run against the configured Kubernetes cluster in ~/.kube/config
-run: generate fmt vet manifests
-	go run ./main.go --cluster-id=abcd --cluster-api-url=https://api.abcd:443 --cert-dir config/examples/certs --metal-api-url http://api.172.17.0.1.nip.io:8080/metal
+# Run against the mini-lab
+deploy: generate fmt vet manifests manager
+	kubectl apply -k config
+	docker build -f Dockerfile.dev -t fcm .
+	kind --name metal-control-plane load docker-image fcm:latest
+	kubectl patch deployment -n firewall firewall-controller-manager --patch='{"spec":{"template":{"spec":{"containers":[{"name": "firewall-controller-manager","imagePullPolicy":"IfNotPresent","image":"fcm:latest"}]}}}}'
+	kubectl delete pod -n firewall -l app=firewall-controller-manager
 
 # Install CRDs into a cluster
 install: manifests
@@ -50,11 +54,6 @@ install: manifests
 uninstall: manifests
 	kustomize build config/crds | kubectl delete -f -
 
-# Deploy controller in the configured Kubernetes cluster in ~/.kube/config
-deploy: manifests
-	cd config/manager && kustomize edit set image controller=${IMG}
-	kustomize build config | kubectl apply -f -
-
 # Generate manifests e.g. CRD, RBAC etc.
 manifests: controller-gen
 	$(CONTROLLER_GEN) +crd:generateEmbeddedObjectMeta=true paths="./..." +output:dir=config/crds

README.md

@@ -2,45 +2,53 @@
 
 ## Overview
 
-The firewall-controller-manager aka FCM is a collection of controllers which are responsible for managing the lifecycle of metal-stack firewalls in a bare-metal kubernetes cluster. It is roughly inspired by the design of Gardener's Machine Controller Manager and Kubernetes' built-in resources `Deployment`, `ReplicaSet` and `Pod`.
+The firewall-controller-manager (FCM) is a collection of controllers which are responsible for managing the lifecycle of firewalls in a [Gardener](https://gardener.cloud/) shoot cluster for the metal-stack provider.
 
-## Objects
+The FCM is typically deployed into the shoot namespace of a seed cluster. This is done by the [gardener-extension-provider-metal](https://github.com/metal-stack/gardener-extension-provider-metal/).
 
-| Custom ResourceObject | Description                                                                                                                                                   |
-| --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `FirewallDeployment`  | A `FirewallDeployment` contains the spec template of a `Firewall` resource similar to a `Deployment` and implements update strategies like rolling update.    |
-| `FirewallSet`         | A `FirewallSet` is similar to ReplicaSet. It is typically owned by a `FirewallDeployment` and attempts to run the defined replica amount of the `Firewall`(s) |
-| `Firewall`            | A `Firewall` is similar to a `Pod` and has a 1:1 relationship to a firewall in the metal-stack api.                                                           |
-| `FirewallMonitor`     | Deployed into the cluster of the user (shoot cluster), which is useful for monitoring the firewall or user-triggered actions on the firewall.                 |
-
-If significant changes were made to the `FirewallDeployment` – like changing the OS image, machine size or firewall networks – then a new `FirewallSet` is created and the existing `Firewall` will be eventually replaced.
-
-The way how a `Firewall` is replaced can be defined with the `FirewallUpdateStrategy`.
+The design of the FCM is roughly inspired by Gardener's [machine-controller-manager](https://github.com/gardener/machine-controller-manager) and Kubernetes' built-in resources `Deployment`, `ReplicaSet` and `Pod`.
 
 ## Architecture
 
-There are three controllers implemented with the following responsibilities.
+The following table is a summary over the [CRDs](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/) introduced by the FCM:
+
+| Custom Resource Object | Description                                                                                                                                                     |
+| ---------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `FirewallDeployment`   | A `FirewallDeployment` contains the spec template of a `Firewall` resource similar to a `Deployment` and implements update strategies like rolling update.      |
+| `FirewallSet`          | A `FirewallSet` is similar to `ReplicaSet`. It is typically owned by a `FirewallDeployment` and attempts to run the defined replica amount of the `Firewall`(s) |
+| `Firewall`             | A `Firewall` is similar to a `Pod` and has a 1:1 relationship to a firewall in the metal-stack api.                                                             |
+| `FirewallMonitor`      | Deployed into the cluster of the user (shoot cluster), which is useful for monitoring the firewall or user-triggered actions on the firewall.                   |
 
 ### `FirewallDeploymentController`
 
-Reconciles the `FirewallDeployment` which was created and manages the lifecycle of a `FirewallSet`. It creates a ServiceAccount Token for the firewall to be able to talk to the kubernetes-api server. The template spec is validated and if changes were made, it decides if a new `FirewallSet` must be created or if the existing one just needs to be updated. The resource status shows the overall status.
+The `FirewallDeployment` controller manages the lifecycle of `FirewallSet`s. It syncs the `Firewall` template spec and if significant changes were made, it may trigger a `FirewallSet` roll. When choosing `RollingUpdate` as a deployment strategy, the deployment controller is waiting for the firewall-controller to connect before throwing away an old `FirewallSet`. The `Recreate` strategy first releases firewalls before creating a new one (can be useful for environments which ran out of available machines but you still want to update).
+
+The controller also deploys a service account for the firewall-controller to be able to talk to the seed's kube-apiserver.
 
 ### `FirewallSetController`
 
-Creates and deletes `Firewall` objects according to the spec according to the given number of firewall replicas. It also checks the status of the `Firewall` and report that in the own status.
+Creates and deletes `Firewall` objects according to the spec and the given number of firewall replicas. It also checks the status of the `Firewall` and report that in the own status.
 
 ### `FirewallController`
 
-Create and delete the physical firewall machine from the spec at the metal-api.
+Creates and deletes the physical firewall machine from the spec at the [metal-api](https://github.com/metal-stack/metal-api).
 
 ## Rolling a `FirewallSet` through `FirewallMonitor` Annotation
 
 A user can initiate rolling the latest firewall set by annotating a monitor in the following way:
 
 ```bash
-$ kubectl annotate fwmon <firewall-name> firewall.metal-stack.io/roll-set=true
+kubectl annotate fwmon <firewall-name> firewall.metal-stack.io/roll-set=true
 ```
 
-## Deployment
+## Development
+
+Most of the functionality is developed with the help of the [intgration](integration) test suite.
+
+To play with the FCM, you can also run this controller inside the [mini-lab](https://github.com/metal-stack/mini-lab) and without a running Gardener installation:
 
-Firewall Controller Manager must be deployed into the Shoot Namespace in a Seed Cluster if this is a Gardener Managed environment. So the Gardener Extension Provider Metal have to create a appropriate Deployment.
+1. Start up the mini-lab, run `eval $(make dev-env)` and change back to this project's directory
+1. Deploy the FCM into the mini-lab with `make deploy`
+1. Adapt the example [firewalldeployment.yaml](config/examples/firewalldeployment.yaml) and apply with `kubectl apply -f config/examples/firewalldeployment.yaml`
+1. Note that the firewall-controller will not be able to connect to the mini-lab due to network restrictions, so the firewall will not get ready.
+   - You can make the firewall become ready anyway by setting the annotation `kubectl annotate fw <fw-nsme> firewall.metal-stack.io/no-controller-connection=true`

api/v2/defaults/defaults.go

@@ -112,12 +112,12 @@ func (r *firewallDeploymentDefaulter) Default(ctx context.Context, obj runtime.O
 	defaultFirewallSpec(&f.Spec.Template.Spec)
 
 	if f.Spec.Template.Spec.Userdata == "" {
-		err := helper.EnsureFirewallControllerRBAC(ctx, r.c.GetSeedConfig(), f, r.c.GetShootNamespace(), r.c.GetShootAccess(), r.c.GetShootAccessHelper())
+		shootConfig, err := r.c.GetShootAccessHelper().RESTConfig(ctx)
 		if err != nil {
 			return err
 		}
 
-		shootConfig, err := r.c.GetShootAccessHelper().RESTConfig(ctx)
+		err = helper.EnsureFirewallControllerRBAC(ctx, r.c.GetSeedConfig(), shootConfig, f, r.c.GetShootNamespace(), r.c.GetShootAccess())
 		if err != nil {
 			return err
 		}
@@ -131,7 +131,7 @@ func (r *firewallDeploymentDefaulter) Default(ctx context.Context, obj runtime.O
 			ForShoot:     true,
 		})
 		if err != nil {
-			return err
+			return fmt.Errorf("error creating raw shoot kubeconfig: %w", err)
 		}
 
 		seedKubeconfig, err := helper.GetAccessKubeconfig(&helper.AccessConfig{
@@ -142,7 +142,7 @@ func (r *firewallDeploymentDefaulter) Default(ctx context.Context, obj runtime.O
 			Deployment:   f,
 		})
 		if err != nil {
-			return err
+			return fmt.Errorf("error creating raw seed kubeconfig: %w", err)
 		}
 
 		userdata, err := renderUserdata(shootKubeconfig, seedKubeconfig)

api/v2/helper/seed_access.go

@@ -21,13 +21,13 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 )
 
-func EnsureFirewallControllerRBAC(ctx context.Context, seedConfig *rest.Config, deploy *v2.FirewallDeployment, shootNamespace string, shootAccess *v2.ShootAccess, shootAccessHelper *ShootAccessHelper) error {
+func EnsureFirewallControllerRBAC(ctx context.Context, seedConfig, shootConfig *rest.Config, deploy *v2.FirewallDeployment, shootNamespace string, shootAccess *v2.ShootAccess) error {
 	err := ensureSeedRBAC(ctx, seedConfig, deploy, shootAccess)
 	if err != nil {
 		return fmt.Errorf("unable to ensure seed rbac: %w", err)
 	}
 
-	err = ensureShootRBAC(ctx, shootAccessHelper, shootNamespace, deploy)
+	err = ensureShootRBAC(ctx, shootConfig, shootNamespace, deploy)
 	if err != nil {
 		return fmt.Errorf("unable to ensure shoot rbac: %w", err)
 	}
@@ -158,7 +158,7 @@ func ensureSeedRBAC(ctx context.Context, seedConfig *rest.Config, deploy *v2.Fir
 	return nil
 }
 
-func ensureShootRBAC(ctx context.Context, shootAccessHelper *ShootAccessHelper, shootNamespace string, deploy *v2.FirewallDeployment) error {
+func ensureShootRBAC(ctx context.Context, shootConfig *rest.Config, shootNamespace string, deploy *v2.FirewallDeployment) error {
 	var (
 		name           = shootAccessResourceName(deploy)
 		serviceAccount = &corev1.ServiceAccount{
@@ -179,12 +179,14 @@ func ensureShootRBAC(ctx context.Context, shootAccessHelper *ShootAccessHelper,
 		}
 	)
 
-	k8sVersion, err := shootAccessHelper.K8sVersion(ctx)
+	k8sVersion, err := determineK8sVersion(shootConfig)
 	if err != nil {
 		return fmt.Errorf("unable to determine shoot k8s version: %w", err)
 	}
 
-	shoot, err := shootAccessHelper.Client(ctx)
+	shoot, err := controllerclient.New(shootConfig, controllerclient.Options{
+		Scheme: scheme,
+	})
 	if err != nil {
 		return fmt.Errorf("unable to create shoot client: %w", err)
 	}
@@ -204,7 +206,7 @@ func ensureShootRBAC(ctx context.Context, shootAccessHelper *ShootAccessHelper,
 			},
 		}
 
-		_, err := controllerutil.CreateOrUpdate(ctx, shoot, serviceAccount, func() error {
+		_, err := controllerutil.CreateOrUpdate(ctx, shoot, serviceAccountSecret, func() error {
 			serviceAccountSecret.Annotations = map[string]string{
 				"kubernetes.io/service-account.name": serviceAccount.Name,
 			}

api/v2/helper/shoot_access.go

@@ -7,7 +7,6 @@ import (
 	"path"
 	"time"
 
-	"github.com/Masterminds/semver/v3"
 	"github.com/go-logr/logr"
 	v2 "github.com/metal-stack/firewall-controller-manager/api/v2"
 	corev1 "k8s.io/api/core/v1"
@@ -17,8 +16,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	controllerclient "sigs.k8s.io/controller-runtime/pkg/client"
 
-	"github.com/golang-jwt/jwt/v4"
-
 	"k8s.io/client-go/tools/clientcmd"
 	configlatest "k8s.io/client-go/tools/clientcmd/api/latest"
 	configv1 "k8s.io/client-go/tools/clientcmd/api/v1"
@@ -28,16 +25,64 @@ type ShootAccessHelper struct {
 	seed      client.Client
 	access    *v2.ShootAccess
 	tokenPath string
+
+	shootConfig *rest.Config
 }
 
+// NewShootAccessHelper provides shoot access functions based on shoot access secrets,
+// i.e. Gardener's generic kubeconfig and token secret.
 func NewShootAccessHelper(seed client.Client, access *v2.ShootAccess) *ShootAccessHelper {
 	return &ShootAccessHelper{
 		seed:   seed,
 		access: access,
 	}
 }
 
+// NewSingleClusterModeHelper provides shoot access functions when running in a single-mode
+// cluster, i.e. the shoot client equals the seed client.
+func NewSingleClusterModeHelper(shootConfig *rest.Config) *ShootAccessHelper {
+	return &ShootAccessHelper{
+		shootConfig: shootConfig,
+	}
+}
+
 func (s *ShootAccessHelper) Config(ctx context.Context) (*configv1.Config, error) {
+	if s.shootConfig != nil {
+		return &configv1.Config{
+			Kind:       "Config",
+			APIVersion: "v1",
+			Clusters: []configv1.NamedCluster{
+				{
+					Name: "default-cluster",
+					Cluster: configv1.Cluster{
+						Server:                   s.shootConfig.Host,
+						CertificateAuthorityData: s.shootConfig.CAData,
+					},
+				},
+			},
+			Contexts: []configv1.NamedContext{
+				{
+					Name: "default-context",
+					Context: configv1.Context{
+						Cluster:   "default-cluster",
+						Namespace: "default",
+						AuthInfo:  "default",
+					},
+				},
+			},
+			CurrentContext: "default-context",
+			AuthInfos: []configv1.NamedAuthInfo{
+				{
+					Name: "default",
+					AuthInfo: configv1.AuthInfo{
+						Token:     s.shootConfig.BearerToken,
+						TokenFile: s.shootConfig.BearerTokenFile,
+					},
+				},
+			},
+		}, nil
+	}
+
 	kubeconfigTemplate := &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      s.access.GenericKubeconfigSecretName,
@@ -90,6 +135,10 @@ func (s *ShootAccessHelper) Raw(ctx context.Context) ([]byte, error) {
 }
 
 func (s *ShootAccessHelper) RESTConfig(ctx context.Context) (*rest.Config, error) {
+	if s.shootConfig != nil {
+		return s.shootConfig, nil
+	}
+
 	raw, err := s.Raw(ctx)
 	if err != nil {
 		return nil, err
@@ -104,9 +153,18 @@ func (s *ShootAccessHelper) RESTConfig(ctx context.Context) (*rest.Config, error
 }
 
 func (s *ShootAccessHelper) Client(ctx context.Context) (client.Client, error) {
-	config, err := s.RESTConfig(ctx)
-	if err != nil {
-		return nil, err
+	var (
+		config *rest.Config
+		err    error
+	)
+
+	if s.shootConfig != nil {
+		config = s.shootConfig
+	} else {
+		config, err = s.RESTConfig(ctx)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	client, err := controllerclient.New(config, controllerclient.Options{
@@ -119,21 +177,7 @@ func (s *ShootAccessHelper) Client(ctx context.Context) (client.Client, error) {
 	return client, err
 }
 
-func (s *ShootAccessHelper) K8sVersion(ctx context.Context) (*semver.Version, error) {
-	config, err := s.RESTConfig(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	v, err := determineK8sVersion(config)
-	if err != nil {
-		return nil, err
-	}
-
-	return v, err
-}
-
-func (s *ShootAccessHelper) ReadTokenSecret(ctx context.Context) (*time.Time, string, error) {
+func (s *ShootAccessHelper) readTokenSecret(ctx context.Context) (string, error) {
 	tokenSecret := &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      s.access.TokenSecretName,
@@ -143,22 +187,12 @@ func (s *ShootAccessHelper) ReadTokenSecret(ctx context.Context) (*time.Time, st
 
 	err := s.seed.Get(ctx, client.ObjectKeyFromObject(tokenSecret), tokenSecret)
 	if err != nil {
-		return nil, "", fmt.Errorf("unable to read token secret: %w", err)
+		return "", fmt.Errorf("unable to read token secret: %w", err)
 	}
 
 	token := string(tokenSecret.Data["token"])
 
-	claims := &jwt.RegisteredClaims{}
-	_, _, err = new(jwt.Parser).ParseUnverified(token, claims)
-	if err != nil {
-		return nil, "", fmt.Errorf("shoot access token is not parsable: %w", err)
-	}
-
-	if claims.ExpiresAt != nil {
-		return &claims.ExpiresAt.Time, token, nil
-	}
-
-	return nil, token, nil
+	return token, nil
 }
 
 type ShootAccessTokenUpdater struct {
@@ -187,7 +221,7 @@ func (s *ShootAccessTokenUpdater) UpdateContinuously(log logr.Logger, stop conte
 	log.Info("updating token file", "path", s.s.tokenPath)
 
 	ctx, cancel := context.WithTimeout(stop, 3*time.Second)
-	_, token, err := s.s.ReadTokenSecret(ctx)
+	token, err := s.s.readTokenSecret(ctx)
 	cancel()
 	if err != nil {
 		return fmt.Errorf("unable to read token secret: %w", err)
@@ -210,7 +244,7 @@ func (s *ShootAccessTokenUpdater) UpdateContinuously(log logr.Logger, stop conte
 				log.Info("updating token file", "path", s.s.tokenPath)
 
 				ctx, cancel := context.WithTimeout(stop, 3*time.Second)
-				_, token, err := s.s.ReadTokenSecret(ctx)
+				token, err := s.s.readTokenSecret(ctx)
 				cancel()
 				if err != nil {
 					log.Error(err, "unable to read token secret")

config/examples/certs/ca-key.pem

@@ -1,5 +1,5 @@
 -----BEGIN EC PRIVATE KEY-----
-MHcCAQEEIHFtf3FGAuNvLsV9tZ7B8XBN+PbsFSywTicN8DKEXxg/oAoGCCqGSM49
-AwEHoUQDQgAE9+G9URRcWNNJ0AcWvxFPPGvWWP5Ai3+u0wxP9URZ/X6PhiAisNvS
-J0KmQYErp3e5ni4goiZmPWMfbU/9bs8KoA==
+MHcCAQEEIBRabFggNFg6LUPxY5AeplDzeqZQmnsnFY9OmWQW2eGBoAoGCCqGSM49
+AwEHoUQDQgAEkP91tJGv5pIytEgKOlwTeksfWC1MczdEmj8ouOiaQfFvCkLl5NB/
+uRLrjoR8vDamER2UM+BumDy1XfM849aIww==
 -----END EC PRIVATE KEY-----