From f5c9a4b3a1446141cf7b980a586a9c995db1dec0 Mon Sep 17 00:00:00 2001 From: Enrico Candino Date: Wed, 19 Feb 2025 16:27:46 +0100 Subject: [PATCH] fix duplicated envvars, added tests (#260) --- k3k-kubelet/provider/provider.go | 126 +++++++++++++------------- k3k-kubelet/provider/provider_test.go | 70 ++++++++++++++ 2 files changed, 135 insertions(+), 61 deletions(-) create mode 100644 k3k-kubelet/provider/provider_test.go diff --git a/k3k-kubelet/provider/provider.go b/k3k-kubelet/provider/provider.go index e42f95f3..2be518f7 100644 --- a/k3k-kubelet/provider/provider.go +++ b/k3k-kubelet/provider/provider.go @@ -373,11 +373,15 @@ func (p *Provider) createPod(ctx context.Context, pod *corev1.Pod) error { if err := p.transformTokens(ctx, pod, tPod); err != nil { return fmt.Errorf("unable to transform tokens for pod %s/%s: %w", pod.Namespace, pod.Name, err) } + // inject networking information to the pod including the virtual cluster controlplane endpoint - p.configureNetworking(pod.Name, pod.Namespace, tPod, p.serverIP) + configureNetworking(tPod, pod.Name, pod.Namespace, p.serverIP, p.dnsIP) + + p.logger.Infow("creating pod", + "host_namespace", tPod.Namespace, "host_name", tPod.Name, + "virtual_namespace", pod.Namespace, "virtual_name", pod.Name, + ) - p.logger.Infow("Creating pod", "Host Namespace", tPod.Namespace, "Host Name", tPod.Name, - "Virtual Namespace", pod.Namespace, "Virtual Name", "env", pod.Name, pod.Spec.Containers[0].Env) return p.HostClient.Create(ctx, tPod) } @@ -722,75 +726,75 @@ func (p *Provider) GetPods(ctx context.Context) ([]*corev1.Pod, error) { // configureNetworking will inject network information to each pod to connect them to the // virtual cluster api server, as well as confiugre DNS information to connect them to the // synced coredns on the host cluster. -func (p *Provider) configureNetworking(podName, podNamespace string, pod *corev1.Pod, serverIP string) { +func configureNetworking(pod *corev1.Pod, podName, podNamespace, serverIP, dnsIP string) { // inject serverIP to hostalias for the pod - KubernetesHostAlias := corev1.HostAlias{ - IP: serverIP, - Hostnames: []string{"kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", "kubernetes.default.svc.cluster.local"}, - } - pod.Spec.HostAliases = append(pod.Spec.HostAliases, KubernetesHostAlias) - // inject networking information to the pod's environment variables - for i := range pod.Spec.Containers { - pod.Spec.Containers[i].Env = append(pod.Spec.Containers[i].Env, - corev1.EnvVar{ - Name: "KUBERNETES_PORT_443_TCP", - Value: "tcp://" + p.serverIP + ":6443", - }, - corev1.EnvVar{ - Name: "KUBERNETES_PORT", - Value: "tcp://" + p.serverIP + ":6443", - }, - corev1.EnvVar{ - Name: "KUBERNETES_PORT_443_TCP_ADDR", - Value: p.serverIP, - }, - corev1.EnvVar{ - Name: "KUBERNETES_SERVICE_HOST", - Value: p.serverIP, - }, - corev1.EnvVar{ - Name: "KUBERNETES_SERVICE_PORT", - Value: "6443", - }, - ) - } - // handle init containers as well - for i := range pod.Spec.InitContainers { - pod.Spec.InitContainers[i].Env = append(pod.Spec.InitContainers[i].Env, - corev1.EnvVar{ - Name: "KUBERNETES_PORT_443_TCP", - Value: "tcp://" + p.serverIP + ":6443", - }, - corev1.EnvVar{ - Name: "KUBERNETES_PORT", - Value: "tcp://" + p.serverIP + ":6443", - }, - corev1.EnvVar{ - Name: "KUBERNETES_PORT_443_TCP_ADDR", - Value: p.serverIP, - }, - corev1.EnvVar{ - Name: "KUBERNETES_SERVICE_HOST", - Value: p.serverIP, - }, - corev1.EnvVar{ - Name: "KUBERNETES_SERVICE_PORT", - Value: "6443", - }, - ) - } + pod.Spec.HostAliases = append(pod.Spec.HostAliases, corev1.HostAlias{ + IP: serverIP, + Hostnames: []string{ + "kubernetes", + "kubernetes.default", + "kubernetes.default.svc", + "kubernetes.default.svc.cluster", + "kubernetes.default.svc.cluster.local", + }, + }) + // injecting cluster DNS IP to the pods except for coredns pod if !strings.HasPrefix(podName, "coredns") { pod.Spec.DNSPolicy = corev1.DNSNone pod.Spec.DNSConfig = &corev1.PodDNSConfig{ Nameservers: []string{ - p.dnsIP, + dnsIP, }, Searches: []string{ - podNamespace + ".svc.cluster.local", "svc.cluster.local", "cluster.local", + podNamespace + ".svc.cluster.local", + "svc.cluster.local", + "cluster.local", }, } } + + updatedEnvVars := []corev1.EnvVar{ + {Name: "KUBERNETES_PORT", Value: "tcp://" + serverIP + ":6443"}, + {Name: "KUBERNETES_SERVICE_HOST", Value: serverIP}, + {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, + {Name: "KUBERNETES_SERVICE_PORT_HTTPS", Value: "6443"}, + {Name: "KUBERNETES_PORT_443_TCP", Value: "tcp://" + serverIP + ":6443"}, + {Name: "KUBERNETES_PORT_443_TCP_ADDR", Value: serverIP}, + {Name: "KUBERNETES_PORT_443_TCP_PORT", Value: "6443"}, + } + + // inject networking information to the pod's environment variables + for i := range pod.Spec.Containers { + pod.Spec.Containers[i].Env = overrideEnvVars(pod.Spec.Containers[i].Env, updatedEnvVars) + } + + // handle init containers as well + for i := range pod.Spec.InitContainers { + pod.Spec.InitContainers[i].Env = overrideEnvVars(pod.Spec.InitContainers[i].Env, updatedEnvVars) + } + +} + +// overrideEnvVars will override the orig environment variables if found in the updated list +func overrideEnvVars(orig, updated []corev1.EnvVar) []corev1.EnvVar { + if len(updated) == 0 { + return orig + } + + // create map for single lookup + updatedEnvVarMap := make(map[string]corev1.EnvVar) + for _, updatedEnvVar := range updated { + updatedEnvVarMap[updatedEnvVar.Name] = updatedEnvVar + } + + for i, origEnvVar := range orig { + if updatedEnvVar, found := updatedEnvVarMap[origEnvVar.Name]; found { + orig[i] = updatedEnvVar + } + } + + return orig } // getSecretsAndConfigmaps retrieves a list of all secrets/configmaps that are in use by a given pod. Useful diff --git a/k3k-kubelet/provider/provider_test.go b/k3k-kubelet/provider/provider_test.go new file mode 100644 index 00000000..1fc63679 --- /dev/null +++ b/k3k-kubelet/provider/provider_test.go @@ -0,0 +1,70 @@ +package provider + +import ( + "reflect" + "testing" + + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" +) + +func Test_overrideEnvVars(t *testing.T) { + type args struct { + orig []corev1.EnvVar + new []corev1.EnvVar + } + tests := []struct { + name string + args args + want []corev1.EnvVar + }{ + { + name: "orig and new are empty", + args: args{ + orig: []v1.EnvVar{}, + new: []v1.EnvVar{}, + }, + want: []v1.EnvVar{}, + }, + { + name: "only orig is empty", + args: args{ + orig: []v1.EnvVar{}, + new: []v1.EnvVar{{Name: "FOO", Value: "new_val"}}, + }, + want: []v1.EnvVar{}, + }, + { + name: "orig has a matching element", + args: args{ + orig: []v1.EnvVar{{Name: "FOO", Value: "old_val"}}, + new: []v1.EnvVar{{Name: "FOO", Value: "new_val"}}, + }, + want: []v1.EnvVar{{Name: "FOO", Value: "new_val"}}, + }, + { + name: "orig have multiple elements", + args: args{ + orig: []v1.EnvVar{{Name: "FOO_0", Value: "old_val_0"}, {Name: "FOO_1", Value: "old_val_1"}}, + new: []v1.EnvVar{{Name: "FOO_1", Value: "new_val_1"}}, + }, + want: []v1.EnvVar{{Name: "FOO_0", Value: "old_val_0"}, {Name: "FOO_1", Value: "new_val_1"}}, + }, + { + name: "orig and new have multiple elements and some not matching", + args: args{ + orig: []v1.EnvVar{{Name: "FOO_0", Value: "old_val_0"}, {Name: "FOO_1", Value: "old_val_1"}}, + new: []v1.EnvVar{{Name: "FOO_1", Value: "new_val_1"}, {Name: "FOO_2", Value: "val_1"}}, + }, + want: []v1.EnvVar{{Name: "FOO_0", Value: "old_val_0"}, {Name: "FOO_1", Value: "new_val_1"}}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := overrideEnvVars(tt.args.orig, tt.args.new); !reflect.DeepEqual(got, tt.want) { + t.Errorf("overrideEnvVars() = %v, want %v", got, tt.want) + } + }) + } +}