[FEATURE] - Helm Webhooks

Adding the ability for the helm charts to manage the deployment of the validating and mutating webhooks

Makefile

@@ -80,7 +80,8 @@ controller-gen:
 		crd \
 		output:crd:dir=charts/terranetes-controller/crds \
 		webhook \
-		output:webhook:dir=deploy/webhooks
+		output:webhook:dir=deploy/webhooks 
+	@cp deploy/webhooks/manifests.yaml charts/terranetes-controller/templates/webhooks.yaml
 	@./hack/patch-crd-gen.sh
 	@./hack/gofmt.sh pkg/apis/*/*/zz_generated.deepcopy.go
 

charts/terranetes-controller/templates/deployment.yaml

@@ -69,6 +69,7 @@ spec:
             - --enable-terraform-versions={{ .Values.controller.enableTerraformVersions }}
             - --enable-watchers={{ .Values.controller.enableWatchers }}
             - --enable-webhook-prefix={{ .Values.controller.webhooks.prefix }}
+            - --enable-webhooks-registration={{ .Values.controller.enableControllerWebhookRegistration }}
             - --enable-webhooks={{ .Values.controller.webhooks.enabled }}
             - --executor-image={{ .Values.controller.images.executor }}
             {{- range .Values.controller.executorSecrets }}

charts/terranetes-controller/templates/webhooks.yaml

@@ -0,0 +1,240 @@
+{{- if .Values.controller.enableHelmWebhookRegistration }}
+{{- $secret := lookup "v1" "Secret" .Release.Namespace .Values.controller.webhooks.caSecret }}
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: terranetes-controller
+  annotations:
+    "helm.sh/hook": post-delete, post-install
+    "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded
+  labels: 
+    {{- include "terranetes-controller.labels" . | nindent 4 }}
+webhooks:
+  - admissionReviewVersions:
+      - v1
+    clientConfig:
+      caBundle: {{ index $secret.data "ca.pem" }}
+      service:
+        name: controller
+        namespace: {{ .Release.Namespace }}
+        path: /mutate/terraform.appvia.io/cloudresources
+    failurePolicy: Fail
+    name: cloudresources.terraform.appvia.io
+    rules:
+      - apiGroups:
+          - terraform.appvia.io
+        apiVersions:
+          - v1alpha1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - cloudresources
+    sideEffects: None
+  - admissionReviewVersions:
+      - v1
+    clientConfig:
+      caBundle: {{ index $secret.data "ca.pem" }}
+      service:
+        name: controller
+        namespace: {{ .Release.Namespace }}
+        path: /mutate/terraform.appvia.io/configurations
+    failurePolicy: Fail
+    name: configurations.terraform.appvia.io
+    rules:
+      - apiGroups:
+          - terraform.appvia.io
+        apiVersions:
+          - v1alpha1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - configurations
+    sideEffects: None
+  - admissionReviewVersions:
+      - v1
+    clientConfig:
+      caBundle: {{ index $secret.data "ca.pem" }}
+      service:
+        name: controller
+        namespace: {{ .Release.Namespace }}
+        path: /mutate/terraform.appvia.io/revisions
+    failurePolicy: Fail
+    name: revisions.terraform.appvia.io
+    rules:
+      - apiGroups:
+          - terraform.appvia.io
+        apiVersions:
+          - v1alpha1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - revisions
+    sideEffects: None
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: terranetes-controller
+  annotations:
+    "helm.sh/hook": post-delete, post-install
+    "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded
+  labels: 
+    {{- include "terranetes-controller.labels" . | nindent 4 }}
+webhooks:
+  {{- if .Values.controller.enableNamespaceProtection }}
+  - admissionReviewVersions:
+      - v1
+    clientConfig:
+      caBundle: {{ index $secret.data "ca.pem" }}
+      service:
+        name: controller
+        namespace: {{ .Release.Namespace }}
+        path: /validate/terraform.appvia.io/namespaces
+    failurePolicy: Fail
+    name: cloudresources.terraform.appvia.io
+    rules:
+      - apiGroups:
+          - ""
+        apiVersions:
+          - v1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - namespaces
+    sideEffects: None
+  {{- end }}
+  - admissionReviewVersions:
+      - v1
+    clientConfig:
+      caBundle: {{ index $secret.data "ca.pem" }}
+      service:
+        name: controller
+        namespace: {{ .Release.Namespace }}
+        path: /validate/terraform.appvia.io/cloudresources
+    failurePolicy: Fail
+    name: cloudresources.terraform.appvia.io
+    rules:
+      - apiGroups:
+          - terraform.appvia.io
+        apiVersions:
+          - v1alpha1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - cloudresources
+    sideEffects: None
+  - admissionReviewVersions:
+      - v1
+    clientConfig:
+      caBundle: {{ index $secret.data "ca.pem" }}
+      service:
+        name: controller
+        namespace: {{ .Release.Namespace }}
+        path: /validate/terraform.appvia.io/configurations
+    failurePolicy: Fail
+    name: configurations.terraform.appvia.io
+    rules:
+      - apiGroups:
+          - terraform.appvia.io
+        apiVersions:
+          - v1alpha1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - configurations
+    sideEffects: None
+  - admissionReviewVersions:
+      - v1
+    clientConfig:
+      caBundle: {{ index $secret.data "ca.pem" }}
+      service:
+        name: controller
+        namespace: {{ .Release.Namespace }}
+        path: /validate/terraform.appvia.io/contexts
+    failurePolicy: Fail
+    name: contexts.terraform.appvia.io
+    rules:
+      - apiGroups:
+          - terraform.appvia.io
+        apiVersions:
+          - v1alpha1
+        operations:
+          - CREATE
+          - DELETE
+          - UPDATE
+        resources:
+          - contexts
+    sideEffects: None
+  - admissionReviewVersions:
+      - v1
+    clientConfig:
+      caBundle: {{ index $secret.data "ca.pem" }}
+      service:
+        name: controller
+        namespace: {{ .Release.Namespace }}
+        path: /validate/terraform.appvia.io/policies
+    failurePolicy: Fail
+    name: policies.terraform.appvia.io
+    rules:
+      - apiGroups:
+          - terraform.appvia.io
+        apiVersions:
+          - v1alpha1
+        operations:
+          - CREATE
+          - DELETE
+          - UPDATE
+        resources:
+          - policies
+    sideEffects: None
+  - admissionReviewVersions:
+      - v1
+    clientConfig:
+      caBundle: {{ index $secret.data "ca.pem" }}
+      service:
+        name: controller
+        namespace: {{ .Release.Namespace }}
+        path: /validate/terraform.appvia.io/providers
+    failurePolicy: Fail
+    name: providers.terraform.appvia.io
+    rules:
+      - apiGroups:
+          - terraform.appvia.io
+        apiVersions:
+          - v1alpha1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - providers
+    sideEffects: None
+  - admissionReviewVersions:
+      - v1
+    clientConfig:
+      caBundle: {{ index $secret.data "ca.pem" }}
+      service:
+        name: controller
+        namespace: {{ .Release.Namespace }}
+        path: /validate/terraform.appvia.io/revisions
+    failurePolicy: Fail
+    name: revisions.terraform.appvia.io
+    rules:
+      - apiGroups:
+          - terraform.appvia.io
+        apiVersions:
+          - v1alpha1
+        operations:
+          - CREATE
+          - DELETE
+          - UPDATE
+        resources:
+          - revisions
+    sideEffects: None
+{{- end }}

charts/terranetes-controller/values.yaml

@@ -71,11 +71,16 @@ controller:
   # indicate we create the watcher jobs in user namespace, these allow users
   # to view the terraform output
   enableWatchers: true
+  ## Indicates we should forgo the controller registering it's own webhooks and allowing
+  ## helm to manage the webhooks for us
+  enableHelmWebhookRegistration: false
+  ## indicates if the controller should register the valdiation and mutation webhooks
+  ## for the Configuration, Revision and CloudResource resources
+  enableControllerWebhookRegistration: true
   # indicates if namespace deletion should be blocked if the namespace contains one
   # or more Configuration resources, forcing the user to delete correctly
   enableNamespaceProtection: false
-  # indicates if the controller should deny updates to Revisions which are currently
-  # in use
+  # indicates if the controller should deny updates to Revisions which are currently in use
   enableRevisionUpdateProtection: true
   # enableTerraformVersions indicates configurations are permitted to override
   # the terraform version in their spec.

cmd/controller/main.go

@@ -56,35 +56,36 @@ func main() {
 
 	flags := cmd.Flags()
 	flags.Bool("verbose", false, "Enable verbose logging")
-	flags.IntVar(&config.BackoffLimit, "backoff-limit", 1, "The number of times we are willing to allow a terraform job to error before marking as a failure")
 	flags.BoolVar(&config.EnableContextInjection, "enable-context-injection", false, "Indicates the controller should inject Configuration context into the terraform variables")
 	flags.BoolVar(&config.EnableNamespaceProtection, "enable-namespace-protection", false, "Indicates the controller should protect the controller namespace from being deleted")
 	flags.BoolVar(&config.EnableRevisionUpdateProtection, "enable-revision-update-protection", false, "Indicates we should protect the revisions in use from being updated")
 	flags.BoolVar(&config.EnableTerraformVersions, "enable-terraform-versions", true, "Indicates the terraform version can be overridden by configurations")
 	flags.BoolVar(&config.EnableWatchers, "enable-watchers", true, "Indicates we create watcher jobs in the configuration namespaces")
+	flags.BoolVar(&config.EnableWebhookPrefix, "enable-webhook-prefix", false, "Indicates the controller should prefix webhook configuration names with the controller name")
 	flags.BoolVar(&config.EnableWebhooks, "enable-webhooks", true, "Indicates we should register the webhooks")
+	flags.BoolVar(&config.EnableWebhooksRegistration, "enable-webhooks-registration", true, "Indicates the controller should register the webhooks for the controller")
 	flags.BoolVar(&config.RegisterCRDs, "register-crds", true, "Indicates the controller to register its own CRDs")
-	flags.BoolVar(&config.EnableWebhookPrefix, "enable-webhook-prefix", false, "Indicates the controller should prefix webhook configuration names with the controller name")
 	flags.DurationVar(&config.DriftControllerInterval, "drift-controller-interval", 5*time.Minute, "Is the check interval for the controller to search for configurations which should be checked for drift")
 	flags.DurationVar(&config.DriftInterval, "drift-interval", 3*time.Hour, "The minimum duration the controller will wait before triggering a drift check")
 	flags.DurationVar(&config.ResyncPeriod, "resync-period", 5*time.Hour, "The resync period for the controller")
 	flags.DurationVar(&config.RevisionExpiration, "revision-expiration", 0, "The duration a revision should be kept is not referenced or latest (zero means disabled)")
 	flags.Float64Var(&config.ConfigurationThreshold, "configurations-threshold", 0, "The maximum percentage of configurations that can be run at any one time")
 	flags.Float64Var(&config.DriftThreshold, "drift-threshold", 0.10, "The maximum percentage of configurations that can be run drift detection at any one time")
 	flags.IntVar(&config.APIServerPort, "apiserver-port", 10080, "The port the apiserver should be listening on")
+	flags.IntVar(&config.BackoffLimit, "backoff-limit", 1, "The number of times we are willing to allow a terraform job to error before marking as a failure")
 	flags.IntVar(&config.MetricsPort, "metrics-port", 9090, "The port the metric endpoint binds to")
 	flags.IntVar(&config.WebhookPort, "webhooks-port", 10081, "The port the webhook endpoint binds to")
 	flags.StringSliceVar(&config.ExecutorSecrets, "executor-secret", []string{}, "Name of a secret in controller namespace which should be added to the job")
-	flags.StringVar(&config.ExecutorMemoryRequest, "executor-memory-request", "32Mi", "The default memory request for the executor container")
-	flags.StringVar(&config.ExecutorMemoryLimit, "executor-memory-limit", "", "The default memory limit for the executor container (default is no limit)")
-	flags.StringVar(&config.ExecutorCPURequest, "executor-cpu-request", "5m", "The default CPU request for the executor container")
-	flags.StringVar(&config.ExecutorCPULimit, "executor-cpu-limit", "", "The default CPU limit for the executor container (default is no limit)")
+	flags.StringSliceVar(&config.JobLabels, "job-label", []string{}, "A collection of key=values to add to all jobs")
 	flags.StringVar(&config.BackendTemplate, "backend-template", "", "Name of secret in the controller namespace containing a template for the terraform state")
+	flags.StringVar(&config.ExecutorCPULimit, "executor-cpu-limit", "", "The default CPU limit for the executor container (default is no limit)")
+	flags.StringVar(&config.ExecutorCPURequest, "executor-cpu-request", "5m", "The default CPU request for the executor container")
 	flags.StringVar(&config.ExecutorImage, "executor-image", fmt.Sprintf("ghcr.io/appvia/terranetes-executor:%s", version.Version), "The image to use for the executor")
+	flags.StringVar(&config.ExecutorMemoryLimit, "executor-memory-limit", "", "The default memory limit for the executor container (default is no limit)")
+	flags.StringVar(&config.ExecutorMemoryRequest, "executor-memory-request", "32Mi", "The default memory request for the executor container")
 	flags.StringVar(&config.InfracostsImage, "infracost-image", "infracosts/infracost:latest", "The image to use for the infracosts")
 	flags.StringVar(&config.InfracostsSecretName, "cost-secret", "", "Name of the secret on the controller namespace containing your infracost token")
 	flags.StringVar(&config.JobTemplate, "job-template", "", "Name of configmap in the controller namespace containing a template for the job")
-	flags.StringSliceVar(&config.JobLabels, "job-label", []string{}, "A collection of key=values to add to all jobs")
 	flags.StringVar(&config.Namespace, "namespace", os.Getenv("KUBE_NAMESPACE"), "The namespace the controller is running in and where jobs will run")
 	flags.StringVar(&config.PolicyImage, "policy-image", "bridgecrew/checkov:latest", "The image to use for the policy")
 	flags.StringVar(&config.PreloadImage, "preload-image", fmt.Sprintf("ghcr.io/appvia/terranetes-executor:%s", version.Version), "The image to use for the preload")

pkg/server/server.go

@@ -274,7 +274,9 @@ func New(cfg *rest.Config, config Config) (*Server, error) {
 
 // Start is called to begin the service
 func (s *Server) Start(ctx context.Context) error {
-	if s.config.EnableWebhooks {
+	if s.config.EnableWebhooks && s.config.EnableWebhooksRegistration {
+		log.Info("registering the terranetes controller webhooks")
+
 		if err := s.registerWebhooks(ctx); err != nil {
 			return fmt.Errorf("failed to register the webhooks, error: %w", err)
 		}

pkg/server/types.go

@@ -51,6 +51,9 @@ type Config struct {
 	EnableNamespaceProtection bool
 	// EnableWebhooks enables the webhooks registration
 	EnableWebhooks bool
+	// EnableWebhooksRegistration indicates the controller should register the webhooks 
+	// for the controller 
+	EnableWebhooksRegistration bool 
 	// EnableWatchers enables the creation of watcher jobs
 	EnableWatchers bool
 	// EnableTerraformVersions indicates if configurations can override the default terraform version

test/e2e/integration/setup.bats

@@ -39,6 +39,8 @@ teardown() {
     cat << EOF > ${BATS_TMPDIR}/my_values.yaml
 replicaCount: 2
 controller:
+  enableControllerWebhookRegistration: false
+  enableHelmWebhookRegistration: true 
   enableNamespaceProtection: true
   images:
     controller: "ghcr.io/appvia/terranetes-controller:${VERSION}"
@@ -52,6 +54,8 @@ EOF
 
     cat << EOF > ${BATS_TMPDIR}/my_values.yaml
 controller:
+  enableControllerWebhookRegistration: false
+  enableHelmWebhookRegistration: true 
   enableNamespaceProtection: true
   costs:
     secret: infracost-api