Add support for pod Wireguard sidecar

Makefile

@@ -27,8 +27,9 @@ KIND_VERSION ?= v0.19.0
 
 
 # images
-AGENT_IMAGE ?= "agent:dev"
-MANAGER_IMAGE ?= "manager:dev"
+AGENT_IMAGE ?= "ghcr.io/jodevsa/wireguard-operator/agent:main"
+MANAGER_IMAGE ?= "ghcr.io/jodevsa/wireguard-operator/manager:main"
+SIDECAR_IMAGE ?= "ghcr.io/jodevsa/wireguard-operator/sidecar:main"
 
 # CHANNELS define the bundle channels used in the bundle.
 # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable")
@@ -123,26 +124,30 @@ build-manager: generate fmt vet ## Build manager binary.
 run: manifests generate fmt vet ## Run a controller from your host.
 	go run ./cmd/manager/main.go
 
-
-
 docker-build-agent:  ## Build docker image with the manager.
 	docker build -t ${AGENT_IMAGE} . -f ./images/agent/Dockerfile
 
 docker-build-manager:  ## Build docker image with the manager.
 	docker build -t ${MANAGER_IMAGE} . -f ./images/manager/Dockerfile
 
-docker-build-integration-test:  docker-build-manager
+docker-build-sidecar:  ## Build docker image with the sidecar.
+	docker build -t ${SIDECAR_IMAGE} . -f ./images/sidecar/Dockerfile
+
+docker-build-all:
 	$(MAKE) docker-build-agent
 	$(MAKE) docker-build-manager
+	${MAKE} docker-build-sidecar
 
+docker-load-kind:
+	kind load docker-image ${AGENT_IMAGE} ${SIDECAR_IMAGE} ${MANAGER_IMAGE}
 
 run-e2e:
-	AGENT_IMAGE=${AGENT_IMAGE} $(MAKE) update-agent-image
+	SIDECAR_IMAGE=${SIDECAR_IMAGE} AGENT_IMAGE=${AGENT_IMAGE} $(MAKE) update-agent-and-sidecar-image
 	MANAGER_IMAGE=${MANAGER_IMAGE} $(MAKE) update-manager-image
-	$(KUSTOMIZE) build config/default > release_it.yaml
+	$(KUSTOMIZE) build config/e2e > release_it.yaml
 	git checkout ./config/default/manager_auth_proxy_patch.yaml
 	git checkout ./config/manager/kustomization.yaml
-	KUBE_CONFIG=$(HOME)/.kube/config KIND_BIN=${KIND} WIREGUARD_OPERATOR_RELEASE_PATH="../../release_it.yaml" AGENT_IMAGE=${AGENT_IMAGE} MANAGER_IMAGE=${MANAGER_IMAGE} go test ./internal/it/ -v -count=1
+	KUBE_CONFIG=$(HOME)/.kube/config KIND_BIN=${KIND} WIREGUARD_OPERATOR_RELEASE_PATH="../../release_it.yaml" SIDECAR_IMAGE=${SIDECAR_IMAGE} AGENT_IMAGE=${AGENT_IMAGE} MANAGER_IMAGE=${MANAGER_IMAGE} go test ./internal/it/ -v -count=1
 
 docker-push: ## Push docker image with the manager.
 	docker push ${IMG}
@@ -156,9 +161,9 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified
 	$(KUSTOMIZE) build config/crd | kubectl delete -f -
 
 
-update-agent-image: kustomize
+update-agent-and-sidecar-image: kustomize
 	## TODO: Simplify later
-	AGENT_IMAGE=$(AGENT_IMAGE) envsubst < ./config/default/manager_auth_proxy_patch.yaml.template > ./config/default/manager_auth_proxy_patch.yaml
+	SIDECAR_IMAGE=$(SIDECAR_IMAGE) AGENT_IMAGE=$(AGENT_IMAGE) envsubst < ./config/default/manager_auth_proxy_patch.yaml.template > ./config/default/manager_auth_proxy_patch.yaml
 
 update-manager-image: kustomize
 	$(info MANAGER_IMAGE: "$(MANAGER_IMAGE)")
@@ -170,7 +175,6 @@ generate-release-file: kustomize update-agent-image update-manager-image
 	git checkout ./config/manager/kustomization.yaml
 
 deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
-	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
 	$(KUSTOMIZE) build config/default | kubectl apply -f -
 
 undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config.

cmd/manager/main.go

@@ -19,10 +19,11 @@ package main
 import (
 	"flag"
 	"fmt"
+	"os"
+
 	vpnv1alpha1 "github.com/jodevsa/wireguard-operator/pkg/api/v1alpha1"
 	"github.com/jodevsa/wireguard-operator/pkg/controllers"
 	v1 "k8s.io/api/core/v1"
-	"os"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
@@ -50,18 +51,22 @@ func init() {
 }
 
 func main() {
-	var agentImagePullPolicy string
+	var wgImage string
+	var wgAgentImagePullPolicy string
+	var wgSidecarImage string
+	var wgSidecarImagePullPolicy string
 	var metricsAddr string
 	var enableLeaderElection bool
 	var probeAddr string
-	var wgImage string
-	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
-	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
-	flag.StringVar(&wgImage, "agent-image", "", "The image used for wireguard server")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
-	flag.StringVar(&agentImagePullPolicy, "agent-image-pull-policy", "IfNotPresent", "Use userspace implementation")
+	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
+	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
+	flag.StringVar(&wgSidecarImage, "sidecar-image", "ghcr.io/jodevsa/wireguard-operator/sidecar:latest", "The image used for wireguard sidecar")
+	flag.StringVar(&wgSidecarImagePullPolicy, "sidecar-image-pull-policy", "IfNotPresent", "imagePullPolicy for wireguard sidecar")
+	flag.StringVar(&wgImage, "agent-image", "ghcr.io/jodevsa/wireguard-operator/agent:latest", "The image used for wireguard server")
+	flag.StringVar(&wgAgentImagePullPolicy, "agent-image-pull-policy", "IfNotPresent", "Use userspace implementation")
 	opts := zap.Options{
 		Development: true,
 	}
@@ -92,7 +97,7 @@ func main() {
 		Client:               mgr.GetClient(),
 		Scheme:               mgr.GetScheme(),
 		AgentImage:           wgImage,
-		AgentImagePullPolicy: v1.PullPolicy(agentImagePullPolicy),
+		AgentImagePullPolicy: v1.PullPolicy(wgAgentImagePullPolicy),
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Wireguard")
 		os.Exit(1)
@@ -104,6 +109,15 @@ func main() {
 		setupLog.Error(err, "unable to create controller", "controller", "WireguardPeer")
 		os.Exit(1)
 	}
+	if err = (&controllers.WireguardSidecarReconciler{
+		Client:                 mgr.GetClient(),
+		Scheme:                 mgr.GetScheme(),
+		SidecarImage:           wgSidecarImage,
+		SidecarImagePullPolicy: v1.PullPolicy(wgSidecarImagePullPolicy),
+	}).SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "WireguardSidecar")
+		os.Exit(1)
+	}
 	//+kubebuilder:scaffold:builder
 
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {

config/default/kustomization.yaml

@@ -44,7 +44,7 @@ patchesStrategicMerge:
 #- webhookcainjection_patch.yaml
 
 # the following config is for teaching kustomize how to do var substitution
-vars:
+vars: []
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
 #- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR
 #  objref:

config/default/manager_auth_proxy_patch.yaml

@@ -33,3 +33,4 @@ spec:
         - "--metrics-bind-address=127.0.0.1:8080"
         - "--leader-elect"
         - "--agent-image=ghcr.io/jodevsa/wireguard-operator/agent:main"
+        - "--sidecar-image=ghcr.io/jodevsa/wireguard-operator/sidecar:main"

config/default/manager_auth_proxy_patch.yaml.template

@@ -33,3 +33,4 @@ spec:
         - "--metrics-bind-address=127.0.0.1:8080"
         - "--leader-elect"
         - "--agent-image=${AGENT_IMAGE}"
+        - "--sidecar-image=${SIDECAR_IMAGE}"

config/e2e/kustomization.yaml

@@ -0,0 +1,20 @@
+# Adds namespace to all resources.
+namespace: wireguard-system
+
+# Value of this field is prepended to the
+# names of all resources, e.g. a deployment named
+# "wordpress" becomes "alices-wordpress".
+# Note that it should also match with the prefix (text before '-') of the namespace
+# field above.
+namePrefix: wireguard-
+
+# Labels to add to all resources and selectors.
+#commonLabels:
+#  someName: someValue
+
+bases:
+- ../default
+
+resources:
+- sidecar-test-deployment.yaml
+- sidecar-test-service.yaml

config/e2e/sidecar-test-deployment.yaml

@@ -0,0 +1,22 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hello-kubernetes
+  annotations:
+    vpn.example.com/enable-sidecar: "true"
+    vpn.example.com/sidecar-wireguard-ref: abcde12345
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: hello-kubernetes
+  template:
+    metadata:
+      labels:
+        app: hello-kubernetes
+    spec:
+      containers:
+        - name: hello-kubernetes
+          image: paulbouwer/hello-kubernetes
+          ports:
+          - containerPort: 8080

config/e2e/sidecar-test-service.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: hello-kubernetes
+spec:
+  selector:
+    app: hello-kubernetes
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 8080
+  type: ClusterIP

images/sidecar/Dockerfile

@@ -0,0 +1,8 @@
+FROM alpine:latest
+
+RUN apk --no-cache add wireguard-tools iptables
+
+COPY images/sidecar/start-wireguard.sh /
+RUN chmod +x /start-wireguard.sh
+
+CMD ["/start-wireguard.sh"]

images/sidecar/start-wireguard.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+set -e
+
+# Generate WireGuard keys
+umask 077
+
+# Start WireGuard
+wg-quick up wg0
+
+# Configure iptables to route traffic over the VPN
+iptables -A FORWARD -i eth0 -o wg0 -j ACCEPT
+iptables -A FORWARD -i wg0 -o eth0 -j ACCEPT
+iptables -t nat -A POSTROUTING -o wg0 -j MASQUERADE
+
+# Start the main container process
+exec "$@"

internal/it/suite_test.go

@@ -3,6 +3,13 @@ package it
 import (
 	"context"
 	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"strings"
+	"testing"
+	"time"
+
 	"github.com/go-logr/stdr"
 	"github.com/jodevsa/wireguard-operator/pkg/api/v1alpha1"
 	. "github.com/onsi/ginkgo"
@@ -12,17 +19,11 @@ import (
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
-	"log"
-	"os"
-	"os/exec"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
 	"sigs.k8s.io/kind/pkg/apis/config/v1alpha4"
 	kind "sigs.k8s.io/kind/pkg/cluster"
 	log2 "sigs.k8s.io/kind/pkg/log"
-	"strings"
-	"testing"
-	"time"
 	//+kubebuilder:scaffold:imports
 )
 
@@ -34,6 +35,7 @@ var k8sClient client.Client
 var testEnv *envtest.Environment
 var releasePath string
 var agentImage string
+var sidecarImage string
 var managerImage string
 var kindBinary string
 var kubeConfigPath string
@@ -116,12 +118,14 @@ func KubectlApply(resource string, namespace string) (string, error) {
 var _ = BeforeSuite(func() {
 	releasePath = os.Getenv("WIREGUARD_OPERATOR_RELEASE_PATH")
 	agentImage = os.Getenv("AGENT_IMAGE")
+	sidecarImage = os.Getenv("SIDECAR_IMAGE")
 	managerImage = os.Getenv("MANAGER_IMAGE")
 	kindBinary = os.Getenv("KIND_BIN")
 	kubeConfigPath = os.Getenv("KUBE_CONFIG")
 
 	Expect(releasePath).NotTo(Equal(""))
 	Expect(agentImage).NotTo(Equal(""))
+	Expect(sidecarImage).NotTo(Equal(""))
 	Expect(releasePath).NotTo(Equal(""))
 	Expect(managerImage).NotTo(Equal(""))
 	Expect(kindBinary).NotTo(Equal(""))
@@ -184,7 +188,13 @@ var _ = BeforeSuite(func() {
 	cmd = exec.Command(kindBinary, "load", "docker-image", agentImage, "--name", testClusterName)
 	b, err = cmd.Output()
 	if err != nil {
-		log.Error(err, "unable to load local image agent:dev")
+		log.Error(err, "unable to load local image for agent")
+		return
+	}
+	cmd = exec.Command(kindBinary, "load", "docker-image", sidecarImage, "--name", testClusterName)
+	b, err = cmd.Output()
+	if err != nil {
+		log.Error(err, "unable to load local image for sidecar")
 		return
 	}
 
@@ -201,17 +211,19 @@ var _ = BeforeSuite(func() {
 		"namespace/wireguard-system",
 		"customresourcedefinition.apiextensions.k8s.io/wireguardpeers.vpn.example.com",
 		"customresourcedefinition.apiextensions.k8s.io/wireguards.vpn.example.com",
-		"serviceaccount/wireguard-controller-manager",
-		"role.rbac.authorization.k8s.io/wireguard-leader-election-role",
-		"clusterrole.rbac.authorization.k8s.io/wireguard-manager-role",
-		"clusterrole.rbac.authorization.k8s.io/wireguard-metrics-reader",
-		"clusterrole.rbac.authorization.k8s.io/wireguard-proxy-role",
-		"rolebinding.rbac.authorization.k8s.io/wireguard-leader-election-rolebinding",
-		"clusterrolebinding.rbac.authorization.k8s.io/wireguard-manager-rolebinding",
-		"clusterrolebinding.rbac.authorization.k8s.io/wireguard-proxy-rolebinding",
-		"configmap/wireguard-manager-config",
-		"service/wireguard-controller-manager-metrics-service",
-		"deployment.apps/wireguard-controller-manager",
+		"serviceaccount/wireguard-wireguard-controller-manager",
+		"role.rbac.authorization.k8s.io/wireguard-wireguard-leader-election-role",
+		"clusterrole.rbac.authorization.k8s.io/wireguard-wireguard-manager-role",
+		"clusterrole.rbac.authorization.k8s.io/wireguard-wireguard-metrics-reader",
+		"clusterrole.rbac.authorization.k8s.io/wireguard-wireguard-proxy-role",
+		"rolebinding.rbac.authorization.k8s.io/wireguard-wireguard-leader-election-rolebinding",
+		"clusterrolebinding.rbac.authorization.k8s.io/wireguard-wireguard-manager-rolebinding",
+		"clusterrolebinding.rbac.authorization.k8s.io/wireguard-wireguard-proxy-rolebinding",
+		"configmap/wireguard-wireguard-manager-config",
+		"service/wireguard-hello-kubernetes",
+		"service/wireguard-wireguard-controller-manager-metrics-service",
+		"deployment.apps/wireguard-hello-kubernetes",
+		"deployment.apps/wireguard-wireguard-controller-manager",
 	}
 
 	Expect(strings.Split(strings.Trim(strings.ReplaceAll(string(b), " created", ""), "\n"), "\n")).To(BeEquivalentTo(expectedResources))

pkg/controllers/wireguard_controller.go

@@ -632,10 +632,10 @@ func (r *WireguardReconciler) serviceForWireguard(m *v1alpha1.Wireguard, service
 
 	dep := &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      m.Name + "-svc",
-			Namespace: m.Namespace,
+			Name:        m.Name + "-svc",
+			Namespace:   m.Namespace,
 			Annotations: m.Spec.ServiceAnnotations,
-			Labels: labels,
+			Labels:      labels,
 		},
 		Spec: corev1.ServiceSpec{
 			Selector: labels,