diff --git a/Makefile b/Makefile index d92448c1..e8885417 100644 --- a/Makefile +++ b/Makefile @@ -27,8 +27,9 @@ KIND_VERSION ?= v0.19.0 # images -AGENT_IMAGE ?= "agent:dev" -MANAGER_IMAGE ?= "manager:dev" +AGENT_IMAGE ?= "ghcr.io/jodevsa/wireguard-operator/agent:main" +MANAGER_IMAGE ?= "ghcr.io/jodevsa/wireguard-operator/manager:main" +SIDECAR_IMAGE ?= "ghcr.io/jodevsa/wireguard-operator/sidecar:main" # CHANNELS define the bundle channels used in the bundle. # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") @@ -123,26 +124,30 @@ build-manager: generate fmt vet ## Build manager binary. run: manifests generate fmt vet ## Run a controller from your host. go run ./cmd/manager/main.go - - docker-build-agent: ## Build docker image with the manager. docker build -t ${AGENT_IMAGE} . -f ./images/agent/Dockerfile docker-build-manager: ## Build docker image with the manager. docker build -t ${MANAGER_IMAGE} . -f ./images/manager/Dockerfile -docker-build-integration-test: docker-build-manager +docker-build-sidecar: ## Build docker image with the sidecar. + docker build -t ${SIDECAR_IMAGE} . -f ./images/sidecar/Dockerfile + +docker-build-all: $(MAKE) docker-build-agent $(MAKE) docker-build-manager + ${MAKE} docker-build-sidecar +docker-load-kind: + kind load docker-image ${AGENT_IMAGE} ${SIDECAR_IMAGE} ${MANAGER_IMAGE} run-e2e: - AGENT_IMAGE=${AGENT_IMAGE} $(MAKE) update-agent-image + SIDECAR_IMAGE=${SIDECAR_IMAGE} AGENT_IMAGE=${AGENT_IMAGE} $(MAKE) update-agent-and-sidecar-image MANAGER_IMAGE=${MANAGER_IMAGE} $(MAKE) update-manager-image - $(KUSTOMIZE) build config/default > release_it.yaml + $(KUSTOMIZE) build config/e2e > release_it.yaml git checkout ./config/default/manager_auth_proxy_patch.yaml git checkout ./config/manager/kustomization.yaml - KUBE_CONFIG=$(HOME)/.kube/config KIND_BIN=${KIND} WIREGUARD_OPERATOR_RELEASE_PATH="../../release_it.yaml" AGENT_IMAGE=${AGENT_IMAGE} MANAGER_IMAGE=${MANAGER_IMAGE} go test ./internal/it/ -v -count=1 + KUBE_CONFIG=$(HOME)/.kube/config KIND_BIN=${KIND} WIREGUARD_OPERATOR_RELEASE_PATH="../../release_it.yaml" SIDECAR_IMAGE=${SIDECAR_IMAGE} AGENT_IMAGE=${AGENT_IMAGE} MANAGER_IMAGE=${MANAGER_IMAGE} go test ./internal/it/ -v -count=1 docker-push: ## Push docker image with the manager. docker push ${IMG} @@ -156,9 +161,9 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified $(KUSTOMIZE) build config/crd | kubectl delete -f - -update-agent-image: kustomize +update-agent-and-sidecar-image: kustomize ## TODO: Simplify later - AGENT_IMAGE=$(AGENT_IMAGE) envsubst < ./config/default/manager_auth_proxy_patch.yaml.template > ./config/default/manager_auth_proxy_patch.yaml + SIDECAR_IMAGE=$(SIDECAR_IMAGE) AGENT_IMAGE=$(AGENT_IMAGE) envsubst < ./config/default/manager_auth_proxy_patch.yaml.template > ./config/default/manager_auth_proxy_patch.yaml update-manager-image: kustomize $(info MANAGER_IMAGE: "$(MANAGER_IMAGE)") @@ -170,7 +175,6 @@ generate-release-file: kustomize update-agent-image update-manager-image git checkout ./config/manager/kustomization.yaml deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} $(KUSTOMIZE) build config/default | kubectl apply -f - undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 471f7610..b0cd69f5 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -19,10 +19,11 @@ package main import ( "flag" "fmt" + "os" + vpnv1alpha1 "github.com/jodevsa/wireguard-operator/pkg/api/v1alpha1" "github.com/jodevsa/wireguard-operator/pkg/controllers" v1 "k8s.io/api/core/v1" - "os" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -50,18 +51,22 @@ func init() { } func main() { - var agentImagePullPolicy string + var wgImage string + var wgAgentImagePullPolicy string + var wgSidecarImage string + var wgSidecarImagePullPolicy string var metricsAddr string var enableLeaderElection bool var probeAddr string - var wgImage string - flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") - flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") - flag.StringVar(&wgImage, "agent-image", "", "The image used for wireguard server") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") - flag.StringVar(&agentImagePullPolicy, "agent-image-pull-policy", "IfNotPresent", "Use userspace implementation") + flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + flag.StringVar(&wgSidecarImage, "sidecar-image", "ghcr.io/jodevsa/wireguard-operator/sidecar:latest", "The image used for wireguard sidecar") + flag.StringVar(&wgSidecarImagePullPolicy, "sidecar-image-pull-policy", "IfNotPresent", "imagePullPolicy for wireguard sidecar") + flag.StringVar(&wgImage, "agent-image", "ghcr.io/jodevsa/wireguard-operator/agent:latest", "The image used for wireguard server") + flag.StringVar(&wgAgentImagePullPolicy, "agent-image-pull-policy", "IfNotPresent", "Use userspace implementation") opts := zap.Options{ Development: true, } @@ -92,7 +97,7 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), AgentImage: wgImage, - AgentImagePullPolicy: v1.PullPolicy(agentImagePullPolicy), + AgentImagePullPolicy: v1.PullPolicy(wgAgentImagePullPolicy), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Wireguard") os.Exit(1) @@ -104,6 +109,15 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "WireguardPeer") os.Exit(1) } + if err = (&controllers.WireguardSidecarReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + SidecarImage: wgSidecarImage, + SidecarImagePullPolicy: v1.PullPolicy(wgSidecarImagePullPolicy), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "WireguardSidecar") + os.Exit(1) + } //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index dfd0f433..568a4cdc 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -44,7 +44,7 @@ patchesStrategicMerge: #- webhookcainjection_patch.yaml # the following config is for teaching kustomize how to do var substitution -vars: +vars: [] # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. #- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR # objref: diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml index 5324092b..994845c2 100644 --- a/config/default/manager_auth_proxy_patch.yaml +++ b/config/default/manager_auth_proxy_patch.yaml @@ -33,3 +33,4 @@ spec: - "--metrics-bind-address=127.0.0.1:8080" - "--leader-elect" - "--agent-image=ghcr.io/jodevsa/wireguard-operator/agent:main" + - "--sidecar-image=ghcr.io/jodevsa/wireguard-operator/sidecar:main" diff --git a/config/default/manager_auth_proxy_patch.yaml.template b/config/default/manager_auth_proxy_patch.yaml.template index 8b189e6a..821b6d80 100644 --- a/config/default/manager_auth_proxy_patch.yaml.template +++ b/config/default/manager_auth_proxy_patch.yaml.template @@ -33,3 +33,4 @@ spec: - "--metrics-bind-address=127.0.0.1:8080" - "--leader-elect" - "--agent-image=${AGENT_IMAGE}" + - "--sidecar-image=${SIDECAR_IMAGE}" diff --git a/config/e2e/kustomization.yaml b/config/e2e/kustomization.yaml new file mode 100644 index 00000000..c2540e68 --- /dev/null +++ b/config/e2e/kustomization.yaml @@ -0,0 +1,20 @@ +# Adds namespace to all resources. +namespace: wireguard-system + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +# "wordpress" becomes "alices-wordpress". +# Note that it should also match with the prefix (text before '-') of the namespace +# field above. +namePrefix: wireguard- + +# Labels to add to all resources and selectors. +#commonLabels: +# someName: someValue + +bases: +- ../default + +resources: +- sidecar-test-deployment.yaml +- sidecar-test-service.yaml \ No newline at end of file diff --git a/config/e2e/sidecar-test-deployment.yaml b/config/e2e/sidecar-test-deployment.yaml new file mode 100644 index 00000000..c664bd72 --- /dev/null +++ b/config/e2e/sidecar-test-deployment.yaml @@ -0,0 +1,22 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hello-kubernetes + annotations: + vpn.example.com/enable-sidecar: "true" + vpn.example.com/sidecar-wireguard-ref: abcde12345 +spec: + replicas: 1 + selector: + matchLabels: + app: hello-kubernetes + template: + metadata: + labels: + app: hello-kubernetes + spec: + containers: + - name: hello-kubernetes + image: paulbouwer/hello-kubernetes + ports: + - containerPort: 8080 diff --git a/config/e2e/sidecar-test-service.yaml b/config/e2e/sidecar-test-service.yaml new file mode 100644 index 00000000..4955b66f --- /dev/null +++ b/config/e2e/sidecar-test-service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: hello-kubernetes +spec: + selector: + app: hello-kubernetes + ports: + - protocol: TCP + port: 80 + targetPort: 8080 + type: ClusterIP diff --git a/images/sidecar/Dockerfile b/images/sidecar/Dockerfile new file mode 100644 index 00000000..4ae33e24 --- /dev/null +++ b/images/sidecar/Dockerfile @@ -0,0 +1,8 @@ +FROM alpine:latest + +RUN apk --no-cache add wireguard-tools iptables + +COPY images/sidecar/start-wireguard.sh / +RUN chmod +x /start-wireguard.sh + +CMD ["/start-wireguard.sh"] diff --git a/images/sidecar/start-wireguard.sh b/images/sidecar/start-wireguard.sh new file mode 100644 index 00000000..582d5fbf --- /dev/null +++ b/images/sidecar/start-wireguard.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +set -e + +# Generate WireGuard keys +umask 077 + +# Start WireGuard +wg-quick up wg0 + +# Configure iptables to route traffic over the VPN +iptables -A FORWARD -i eth0 -o wg0 -j ACCEPT +iptables -A FORWARD -i wg0 -o eth0 -j ACCEPT +iptables -t nat -A POSTROUTING -o wg0 -j MASQUERADE + +# Start the main container process +exec "$@" diff --git a/internal/it/suite_test.go b/internal/it/suite_test.go index 445583be..0bbeef16 100644 --- a/internal/it/suite_test.go +++ b/internal/it/suite_test.go @@ -3,6 +3,13 @@ package it import ( "context" "fmt" + "log" + "os" + "os/exec" + "strings" + "testing" + "time" + "github.com/go-logr/stdr" "github.com/jodevsa/wireguard-operator/pkg/api/v1alpha1" . "github.com/onsi/ginkgo" @@ -12,17 +19,11 @@ import ( "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" - "log" - "os" - "os/exec" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/kind/pkg/apis/config/v1alpha4" kind "sigs.k8s.io/kind/pkg/cluster" log2 "sigs.k8s.io/kind/pkg/log" - "strings" - "testing" - "time" //+kubebuilder:scaffold:imports ) @@ -34,6 +35,7 @@ var k8sClient client.Client var testEnv *envtest.Environment var releasePath string var agentImage string +var sidecarImage string var managerImage string var kindBinary string var kubeConfigPath string @@ -116,12 +118,14 @@ func KubectlApply(resource string, namespace string) (string, error) { var _ = BeforeSuite(func() { releasePath = os.Getenv("WIREGUARD_OPERATOR_RELEASE_PATH") agentImage = os.Getenv("AGENT_IMAGE") + sidecarImage = os.Getenv("SIDECAR_IMAGE") managerImage = os.Getenv("MANAGER_IMAGE") kindBinary = os.Getenv("KIND_BIN") kubeConfigPath = os.Getenv("KUBE_CONFIG") Expect(releasePath).NotTo(Equal("")) Expect(agentImage).NotTo(Equal("")) + Expect(sidecarImage).NotTo(Equal("")) Expect(releasePath).NotTo(Equal("")) Expect(managerImage).NotTo(Equal("")) Expect(kindBinary).NotTo(Equal("")) @@ -184,7 +188,13 @@ var _ = BeforeSuite(func() { cmd = exec.Command(kindBinary, "load", "docker-image", agentImage, "--name", testClusterName) b, err = cmd.Output() if err != nil { - log.Error(err, "unable to load local image agent:dev") + log.Error(err, "unable to load local image for agent") + return + } + cmd = exec.Command(kindBinary, "load", "docker-image", sidecarImage, "--name", testClusterName) + b, err = cmd.Output() + if err != nil { + log.Error(err, "unable to load local image for sidecar") return } @@ -201,17 +211,19 @@ var _ = BeforeSuite(func() { "namespace/wireguard-system", "customresourcedefinition.apiextensions.k8s.io/wireguardpeers.vpn.example.com", "customresourcedefinition.apiextensions.k8s.io/wireguards.vpn.example.com", - "serviceaccount/wireguard-controller-manager", - "role.rbac.authorization.k8s.io/wireguard-leader-election-role", - "clusterrole.rbac.authorization.k8s.io/wireguard-manager-role", - "clusterrole.rbac.authorization.k8s.io/wireguard-metrics-reader", - "clusterrole.rbac.authorization.k8s.io/wireguard-proxy-role", - "rolebinding.rbac.authorization.k8s.io/wireguard-leader-election-rolebinding", - "clusterrolebinding.rbac.authorization.k8s.io/wireguard-manager-rolebinding", - "clusterrolebinding.rbac.authorization.k8s.io/wireguard-proxy-rolebinding", - "configmap/wireguard-manager-config", - "service/wireguard-controller-manager-metrics-service", - "deployment.apps/wireguard-controller-manager", + "serviceaccount/wireguard-wireguard-controller-manager", + "role.rbac.authorization.k8s.io/wireguard-wireguard-leader-election-role", + "clusterrole.rbac.authorization.k8s.io/wireguard-wireguard-manager-role", + "clusterrole.rbac.authorization.k8s.io/wireguard-wireguard-metrics-reader", + "clusterrole.rbac.authorization.k8s.io/wireguard-wireguard-proxy-role", + "rolebinding.rbac.authorization.k8s.io/wireguard-wireguard-leader-election-rolebinding", + "clusterrolebinding.rbac.authorization.k8s.io/wireguard-wireguard-manager-rolebinding", + "clusterrolebinding.rbac.authorization.k8s.io/wireguard-wireguard-proxy-rolebinding", + "configmap/wireguard-wireguard-manager-config", + "service/wireguard-hello-kubernetes", + "service/wireguard-wireguard-controller-manager-metrics-service", + "deployment.apps/wireguard-hello-kubernetes", + "deployment.apps/wireguard-wireguard-controller-manager", } Expect(strings.Split(strings.Trim(strings.ReplaceAll(string(b), " created", ""), "\n"), "\n")).To(BeEquivalentTo(expectedResources)) diff --git a/pkg/controllers/wireguard_controller.go b/pkg/controllers/wireguard_controller.go index 7b265f4a..b722b111 100644 --- a/pkg/controllers/wireguard_controller.go +++ b/pkg/controllers/wireguard_controller.go @@ -632,10 +632,10 @@ func (r *WireguardReconciler) serviceForWireguard(m *v1alpha1.Wireguard, service dep := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: m.Name + "-svc", - Namespace: m.Namespace, + Name: m.Name + "-svc", + Namespace: m.Namespace, Annotations: m.Spec.ServiceAnnotations, - Labels: labels, + Labels: labels, }, Spec: corev1.ServiceSpec{ Selector: labels, diff --git a/pkg/controllers/wireguardsidecar_controller.go b/pkg/controllers/wireguardsidecar_controller.go new file mode 100644 index 00000000..b1155322 --- /dev/null +++ b/pkg/controllers/wireguardsidecar_controller.go @@ -0,0 +1,171 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + "time" + + "github.com/jodevsa/wireguard-operator/pkg/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +const ( + configMapName = "wireguard-peer-config" + configMapNamespace = "default" + configMapKey = "config" +) + +type WireguardSidecarReconciler struct { + client.Client + Scheme *runtime.Scheme + SidecarImage string + SidecarImagePullPolicy corev1.PullPolicy + RequeueAfter time.Duration +} + +func (r *WireguardSidecarReconciler) SetupWithManager(mgr ctrl.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &corev1.Pod{}, "metadata.annotations", func(rawObj client.Object) []string { + pod := rawObj.(*corev1.Pod) + return []string{pod.ObjectMeta.Annotations["vpn.example.com/enable-sidecar"]} + }); err != nil { + return err + } + + return ctrl.NewControllerManagedBy(mgr). + For(&corev1.Pod{}). + WithEventFilter(predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + oldPod := e.ObjectOld.(*corev1.Pod) + newPod := e.ObjectNew.(*corev1.Pod) + return oldPod.ObjectMeta.Annotations["vpn.example.com/enable-sidecar"] != newPod.ObjectMeta.Annotations["vpn.example.com/enable-sidecar"] + }, + }). + Complete(r) +} + +func (r *WireguardSidecarReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + var pod corev1.Pod + if err := r.Get(ctx, req.NamespacedName, &pod); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + if pod.ObjectMeta.Annotations["vpn.example.com/enable-sidecar"] != "true" { + // Pod does not have the desired annotation implement check and garbage collection logic + return ctrl.Result{}, nil + } + + // Check if a sidecar container already exists in the pod spec + hasSidecar := false + for _, container := range pod.Spec.Containers { + if container.Name == "wireguard-sidecar" { + hasSidecar = true + break + } + } + + if !hasSidecar { + + ref, hasRef := pod.ObjectMeta.Annotations["vpn.example.com/sidecar-wireguard-ref"] + + if !hasRef { + return ctrl.Result{}, fmt.Errorf("%s does not have ref annotation", req.Name) + } + + wireguard := &v1alpha1.Wireguard{} + err := r.Get(context.Background(), types.NamespacedName{Name: ref}, wireguard) + if err != nil { + if errors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("Wireguard resource %s not found", req.Name) + } + return ctrl.Result{}, err + } + + peer := &v1alpha1.WireguardPeer{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-sidecar", pod.Name), + Namespace: pod.Namespace, + }, + Spec: v1alpha1.WireguardPeerSpec{ + WireguardRef: ref, + }, + } + + err = r.Client.Create(context.Background(), peer) + if err != nil { + return ctrl.Result{}, fmt.Errorf("Unable to create peer %s", peer.Name) + } + + // Create the configmap for the peer status config + configMapName := fmt.Sprintf("%s-sidecar", pod.Name) + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configMapName, + Namespace: pod.Namespace, + }, + Data: map[string]string{ + "wg0.conf": peer.Status.Config, + }, + } + + err = r.Client.Create(context.Background(), configMap) + if err != nil { + return ctrl.Result{}, fmt.Errorf("Unable to create configMap %s", configMap.Name) + } + + // Add the configmap volume to the pod spec + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: configMap.Name, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: configMap.Name, + }, + }, + }, + }) + + // Mount the configmap in the sidecar container + pod.Spec.Containers = append(pod.Spec.Containers, corev1.Container{ + Name: "wireguard-sidecar", + Image: r.SidecarImage, + ImagePullPolicy: r.SidecarImagePullPolicy, + VolumeMounts: []corev1.VolumeMount{{ + Name: configMap.Name, + MountPath: "/etc/wireguard/wg0.conf", + SubPath: "wg0.conf", + }}, + }) + + if err := r.Update(ctx, &pod); err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil + } + + return ctrl.Result{}, nil +}