Skip to content

Commit

Permalink
fix duplicated envvars, added tests (#260)
Browse files Browse the repository at this point in the history
  • Loading branch information
@enrichman
enrichman authored Feb 19, 2025
1 parent 65fe7a6 commit f5c9a4b
Show file tree
Hide file tree
Showing 2 changed files with 135 additions and 61 deletions.
126 changes: 65 additions & 61 deletions k3k-kubelet/provider/provider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -373,11 +373,15 @@ func (p *Provider) createPod(ctx context.Context, pod *corev1.Pod) error {
if err := p.transformTokens(ctx, pod, tPod); err != nil {
return fmt.Errorf("unable to transform tokens for pod %s/%s: %w", pod.Namespace, pod.Name, err)
}

// inject networking information to the pod including the virtual cluster controlplane endpoint
p.configureNetworking(pod.Name, pod.Namespace, tPod, p.serverIP)
configureNetworking(tPod, pod.Name, pod.Namespace, p.serverIP, p.dnsIP)

p.logger.Infow("creating pod",
"host_namespace", tPod.Namespace, "host_name", tPod.Name,
"virtual_namespace", pod.Namespace, "virtual_name", pod.Name,
)

p.logger.Infow("Creating pod", "Host Namespace", tPod.Namespace, "Host Name", tPod.Name,
"Virtual Namespace", pod.Namespace, "Virtual Name", "env", pod.Name, pod.Spec.Containers[0].Env)
return p.HostClient.Create(ctx, tPod)
}

Expand Down Expand Up @@ -722,75 +726,75 @@ func (p *Provider) GetPods(ctx context.Context) ([]*corev1.Pod, error) {
// configureNetworking will inject network information to each pod to connect them to the
// virtual cluster api server, as well as confiugre DNS information to connect them to the
// synced coredns on the host cluster.
func (p *Provider) configureNetworking(podName, podNamespace string, pod *corev1.Pod, serverIP string) {
func configureNetworking(pod *corev1.Pod, podName, podNamespace, serverIP, dnsIP string) {
// inject serverIP to hostalias for the pod
KubernetesHostAlias := corev1.HostAlias{
IP: serverIP,
Hostnames: []string{"kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", "kubernetes.default.svc.cluster.local"},
}
pod.Spec.HostAliases = append(pod.Spec.HostAliases, KubernetesHostAlias)
// inject networking information to the pod's environment variables
for i := range pod.Spec.Containers {
pod.Spec.Containers[i].Env = append(pod.Spec.Containers[i].Env,
corev1.EnvVar{
Name: "KUBERNETES_PORT_443_TCP",
Value: "tcp://" + p.serverIP + ":6443",
},
corev1.EnvVar{
Name: "KUBERNETES_PORT",
Value: "tcp://" + p.serverIP + ":6443",
},
corev1.EnvVar{
Name: "KUBERNETES_PORT_443_TCP_ADDR",
Value: p.serverIP,
},
corev1.EnvVar{
Name: "KUBERNETES_SERVICE_HOST",
Value: p.serverIP,
},
corev1.EnvVar{
Name: "KUBERNETES_SERVICE_PORT",
Value: "6443",
},
)
}
// handle init containers as well
for i := range pod.Spec.InitContainers {
pod.Spec.InitContainers[i].Env = append(pod.Spec.InitContainers[i].Env,
corev1.EnvVar{
Name: "KUBERNETES_PORT_443_TCP",
Value: "tcp://" + p.serverIP + ":6443",
},
corev1.EnvVar{
Name: "KUBERNETES_PORT",
Value: "tcp://" + p.serverIP + ":6443",
},
corev1.EnvVar{
Name: "KUBERNETES_PORT_443_TCP_ADDR",
Value: p.serverIP,
},
corev1.EnvVar{
Name: "KUBERNETES_SERVICE_HOST",
Value: p.serverIP,
},
corev1.EnvVar{
Name: "KUBERNETES_SERVICE_PORT",
Value: "6443",
},
)
}
pod.Spec.HostAliases = append(pod.Spec.HostAliases, corev1.HostAlias{
IP: serverIP,
Hostnames: []string{
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local",
},
})

// injecting cluster DNS IP to the pods except for coredns pod
if !strings.HasPrefix(podName, "coredns") {
pod.Spec.DNSPolicy = corev1.DNSNone
pod.Spec.DNSConfig = &corev1.PodDNSConfig{
Nameservers: []string{
p.dnsIP,
dnsIP,
},
Searches: []string{
podNamespace + ".svc.cluster.local", "svc.cluster.local", "cluster.local",
podNamespace + ".svc.cluster.local",
"svc.cluster.local",
"cluster.local",
},
}
}

updatedEnvVars := []corev1.EnvVar{
{Name: "KUBERNETES_PORT", Value: "tcp://" + serverIP + ":6443"},
{Name: "KUBERNETES_SERVICE_HOST", Value: serverIP},
{Name: "KUBERNETES_SERVICE_PORT", Value: "6443"},
{Name: "KUBERNETES_SERVICE_PORT_HTTPS", Value: "6443"},
{Name: "KUBERNETES_PORT_443_TCP", Value: "tcp://" + serverIP + ":6443"},
{Name: "KUBERNETES_PORT_443_TCP_ADDR", Value: serverIP},
{Name: "KUBERNETES_PORT_443_TCP_PORT", Value: "6443"},
}

// inject networking information to the pod's environment variables
for i := range pod.Spec.Containers {
pod.Spec.Containers[i].Env = overrideEnvVars(pod.Spec.Containers[i].Env, updatedEnvVars)
}

// handle init containers as well
for i := range pod.Spec.InitContainers {
pod.Spec.InitContainers[i].Env = overrideEnvVars(pod.Spec.InitContainers[i].Env, updatedEnvVars)
}

}

// overrideEnvVars will override the orig environment variables if found in the updated list
func overrideEnvVars(orig, updated []corev1.EnvVar) []corev1.EnvVar {
if len(updated) == 0 {
return orig
}

// create map for single lookup
updatedEnvVarMap := make(map[string]corev1.EnvVar)
for _, updatedEnvVar := range updated {
updatedEnvVarMap[updatedEnvVar.Name] = updatedEnvVar
}

for i, origEnvVar := range orig {
if updatedEnvVar, found := updatedEnvVarMap[origEnvVar.Name]; found {
orig[i] = updatedEnvVar
}
}

return orig
}

// getSecretsAndConfigmaps retrieves a list of all secrets/configmaps that are in use by a given pod. Useful
Expand Down
70 changes: 70 additions & 0 deletions k3k-kubelet/provider/provider_test.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
package provider

import (
"reflect"
"testing"

corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
)

func Test_overrideEnvVars(t *testing.T) {
type args struct {
orig []corev1.EnvVar
new []corev1.EnvVar
}
tests := []struct {
name string
args args
want []corev1.EnvVar
}{
{
name: "orig and new are empty",
args: args{
orig: []v1.EnvVar{},
new: []v1.EnvVar{},
},
want: []v1.EnvVar{},
},
{
name: "only orig is empty",
args: args{
orig: []v1.EnvVar{},
new: []v1.EnvVar{{Name: "FOO", Value: "new_val"}},
},
want: []v1.EnvVar{},
},
{
name: "orig has a matching element",
args: args{
orig: []v1.EnvVar{{Name: "FOO", Value: "old_val"}},
new: []v1.EnvVar{{Name: "FOO", Value: "new_val"}},
},
want: []v1.EnvVar{{Name: "FOO", Value: "new_val"}},
},
{
name: "orig have multiple elements",
args: args{
orig: []v1.EnvVar{{Name: "FOO_0", Value: "old_val_0"}, {Name: "FOO_1", Value: "old_val_1"}},
new: []v1.EnvVar{{Name: "FOO_1", Value: "new_val_1"}},
},
want: []v1.EnvVar{{Name: "FOO_0", Value: "old_val_0"}, {Name: "FOO_1", Value: "new_val_1"}},
},
{
name: "orig and new have multiple elements and some not matching",
args: args{
orig: []v1.EnvVar{{Name: "FOO_0", Value: "old_val_0"}, {Name: "FOO_1", Value: "old_val_1"}},
new: []v1.EnvVar{{Name: "FOO_1", Value: "new_val_1"}, {Name: "FOO_2", Value: "val_1"}},
},
want: []v1.EnvVar{{Name: "FOO_0", Value: "old_val_0"}, {Name: "FOO_1", Value: "new_val_1"}},
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := overrideEnvVars(tt.args.orig, tt.args.new); !reflect.DeepEqual(got, tt.want) {
t.Errorf("overrideEnvVars() = %v, want %v", got, tt.want)
}
})
}
}

0 comments on commit f5c9a4b

Please sign in to comment.