Update e2e lxd-profile and kube-proxy config (#178)

Juju only supports specific devices. This commit removes the unsupported
ones.
Also, the container cannot modify the sysctl config on LXD, thus we
disable the setting in kube-proxy.

src/k8s/pkg/component/network.go

@@ -3,6 +3,7 @@ package component
 import (
 	"context"
 	"fmt"
+	"log"
 	"net"
 	"strings"
 
@@ -98,7 +99,11 @@ func EnableNetworkComponent(ctx context.Context, s snap.Snap, podCIDR string) er
 			return fmt.Errorf("failed to get mount propagation for %s: %w", p, err)
 		}
 		if p == "private" {
-			if s.OnLXD(ctx) {
+			onLXD, err := s.OnLXD(ctx)
+			if err != nil {
+				log.Printf("failed to check if on lxd: %v", err)
+			}
+			if onLXD {
 				return fmt.Errorf("/sys is not a shared mount on the LXD container, this might be resolved by updating LXD on the host to version 5.0.2 or newer")
 			}
 			return fmt.Errorf("/sys is not a shared mount")

src/k8s/pkg/k8sd/app/hooks_bootstrap.go

@@ -127,7 +127,7 @@ func onBootstrapWorkerNode(s *state.State, encodedToken string) error {
 	if err := setup.KubeletWorker(snap, s.Name(), nodeIP, response.ClusterDNS, response.ClusterDomain, response.CloudProvider); err != nil {
 		return fmt.Errorf("failed to configure kubelet: %w", err)
 	}
-	if err := setup.KubeProxy(snap, s.Name(), response.PodCIDR); err != nil {
+	if err := setup.KubeProxy(s.Context, snap, s.Name(), response.PodCIDR); err != nil {
 		return fmt.Errorf("failed to configure kube-proxy: %w", err)
 	}
 	if err := setup.K8sAPIServerProxy(snap, response.APIServers); err != nil {
@@ -234,7 +234,7 @@ func onBootstrapControlPlane(s *state.State, initConfig map[string]string) error
 	if err := setup.KubeletControlPlane(snap, s.Name(), nodeIP, cfg.Kubelet.ClusterDNS, cfg.Kubelet.ClusterDomain, cfg.Kubelet.CloudProvider); err != nil {
 		return fmt.Errorf("failed to configure kubelet: %w", err)
 	}
-	if err := setup.KubeProxy(snap, s.Name(), cfg.Network.PodCIDR); err != nil {
+	if err := setup.KubeProxy(s.Context, snap, s.Name(), cfg.Network.PodCIDR); err != nil {
 		return fmt.Errorf("failed to configure kube-proxy: %w", err)
 	}
 	if err := setup.KubeControllerManager(snap); err != nil {

src/k8s/pkg/k8sd/app/hooks_join.go

@@ -111,7 +111,7 @@ func onPostJoin(s *state.State, initConfig map[string]string) error {
 	if err := setup.KubeletControlPlane(snap, s.Name(), nodeIP, cfg.Kubelet.ClusterDNS, cfg.Kubelet.ClusterDomain, cfg.Kubelet.CloudProvider); err != nil {
 		return fmt.Errorf("failed to configure kubelet: %w", err)
 	}
-	if err := setup.KubeProxy(snap, s.Name(), cfg.Network.PodCIDR); err != nil {
+	if err := setup.KubeProxy(s.Context, snap, s.Name(), cfg.Network.PodCIDR); err != nil {
 		return fmt.Errorf("failed to configure kube-proxy: %w", err)
 	}
 	if err := setup.KubeControllerManager(snap); err != nil {

src/k8s/pkg/k8sd/setup/kube_proxy.go

@@ -1,22 +1,36 @@
 package setup
 
 import (
+	"context"
 	"fmt"
+	"log"
 	"path"
 
 	"github.com/canonical/k8s/pkg/snap"
 	snaputil "github.com/canonical/k8s/pkg/snap/util"
 )
 
 // KubeProxy configures kube-proxy on the local node.
-func KubeProxy(snap snap.Snap, hostname string, podCIDR string) error {
-	if _, err := snaputil.UpdateServiceArguments(snap, "kube-proxy", map[string]string{
+func KubeProxy(ctx context.Context, snap snap.Snap, hostname string, podCIDR string) error {
+	serviceArgs := map[string]string{
 		"--cluster-cidr":         podCIDR,
 		"--healthz-bind-address": "127.0.0.1",
 		"--hostname-override":    hostname,
 		"--kubeconfig":           path.Join(snap.KubernetesConfigDir(), "proxy.conf"),
 		"--profiling":            "false",
-	}, nil); err != nil {
+	}
+
+	onLXD, err := snap.OnLXD(ctx)
+	if err != nil {
+		log.Printf("failed to check if on lxd: %v", err)
+	}
+	if onLXD {
+		// A container cannot set this sysctl config in LXD. So, we disable it by setting it to "0".
+		// See: https://kubernetes.io/docs/reference/command-line-tools-reference/kube-proxy/
+		serviceArgs["--conntrack-max-per-core"] = "0"
+	}
+
+	if _, err := snaputil.UpdateServiceArguments(snap, "kube-proxy", serviceArgs, nil); err != nil {
 		return fmt.Errorf("failed to render arguments file: %w", err)
 	}
 	return nil

src/k8s/pkg/k8sd/setup/kube_proxy_test.go

@@ -0,0 +1,68 @@
+package setup_test
+
+import (
+	"context"
+	"os"
+	"path"
+	"testing"
+
+	"github.com/canonical/k8s/pkg/k8sd/setup"
+	"github.com/canonical/k8s/pkg/snap/mock"
+	snaputil "github.com/canonical/k8s/pkg/snap/util"
+	. "github.com/onsi/gomega"
+)
+
+func TestKubeProxy(t *testing.T) {
+	g := NewWithT(t)
+
+	dir := t.TempDir()
+
+	s := &mock.Snap{
+		Mock: mock.Mock{
+			KubernetesConfigDir: path.Join(dir, "kubernetes"),
+			ServiceArgumentsDir: path.Join(dir, "args"),
+			OnLXD:               false,
+			UID:                 os.Getuid(),
+			GID:                 os.Getgid(),
+		},
+	}
+
+	g.Expect(setup.EnsureAllDirectories(s)).To(BeNil())
+
+	t.Run("Args", func(t *testing.T) {
+		g.Expect(setup.KubeProxy(context.Background(), s, "myhostname", "10.1.0.0/16")).To(BeNil())
+
+		for key, expectedVal := range map[string]string{
+			"--cluster-cidr":           "10.1.0.0/16",
+			"--healthz-bind-address":   "127.0.0.1",
+			"--hostname-override":      "myhostname",
+			"--kubeconfig":             path.Join(dir, "kubernetes", "proxy.conf"),
+			"--profiling":              "false",
+			"--conntrack-max-per-core": "",
+		} {
+			t.Run(key, func(t *testing.T) {
+				g := NewWithT(t)
+				val, err := snaputil.GetServiceArgument(s, "kube-proxy", key)
+				g.Expect(err).To(BeNil())
+				g.Expect(val).To(Equal(expectedVal))
+			})
+		}
+	})
+
+	s.Mock.OnLXD = true
+	t.Run("ArgsOnLXD", func(t *testing.T) {
+		g.Expect(setup.KubeProxy(context.Background(), s, "myhostname", "10.1.0.0/16")).To(BeNil())
+
+		for key, expectedVal := range map[string]string{
+			"--conntrack-max-per-core": "0",
+		} {
+			t.Run(key, func(t *testing.T) {
+				g := NewWithT(t)
+				val, err := snaputil.GetServiceArgument(s, "kube-proxy", key)
+				g.Expect(err).To(BeNil())
+				g.Expect(val).To(Equal(expectedVal))
+			})
+		}
+	})
+
+}

src/k8s/pkg/snap/interface.go

@@ -10,8 +10,8 @@ import (
 
 // Snap abstracts file system paths and interacting with the k8s services.
 type Snap interface {
-	Strict() bool               // Strict returns true if the snap is installed with strict confinement.
-	OnLXD(context.Context) bool // OnLXD returns true if the host runs on LXD.
+	Strict() bool                        // Strict returns true if the snap is installed with strict confinement.
+	OnLXD(context.Context) (bool, error) // OnLXD returns true if the host runs on LXD.
 
 	UID() int // UID is the user ID to set on config files.
 	GID() int // GID is the group ID to set on config files.

src/k8s/pkg/snap/mock/mock.go

@@ -12,6 +12,7 @@ import (
 type Mock struct {
 	Strict                      bool
 	OnLXD                       bool
+	OnLXDErr                    error
 	UID                         int
 	GID                         int
 	KubernetesConfigDir         string
@@ -77,8 +78,8 @@ func (s *Snap) RestartService(ctx context.Context, name string) error {
 func (s *Snap) Strict() bool {
 	return s.Mock.Strict
 }
-func (s *Snap) OnLXD(context.Context) bool {
-	return s.Mock.OnLXD
+func (s *Snap) OnLXD(context.Context) (bool, error) {
+	return s.Mock.OnLXD, s.Mock.OnLXDErr
 }
 func (s *Snap) UID() int {
 	return s.Mock.UID

src/k8s/pkg/snap/snap.go

@@ -10,6 +10,7 @@ import (
 	"github.com/canonical/k8s/pkg/client/dqlite"
 	"github.com/canonical/k8s/pkg/k8sd/types"
 	"github.com/canonical/k8s/pkg/utils"
+	"github.com/moby/sys/mountinfo"
 	"gopkg.in/yaml.v2"
 	"k8s.io/cli-runtime/pkg/genericclioptions"
 )
@@ -85,8 +86,12 @@ func (s *snap) Strict() bool {
 	return meta.Confinement == "strict"
 }
 
-func (s *snap) OnLXD(ctx context.Context) bool {
-	return s.runCommand(ctx, "grep", "-qa", "container=lxc", "/proc/1/environ") == nil
+func (s *snap) OnLXD(ctx context.Context) (bool, error) {
+	mounts, err := mountinfo.GetMounts(mountinfo.FSTypeFilter("fuse.lxcfs"))
+	if err != nil {
+		return false, fmt.Errorf("failed to check for lxcfs mounts: %w", err)
+	}
+	return len(mounts) > 0, nil
 }
 
 func (s *snap) UID() int {

tests/e2e/lxd-profile.yaml

@@ -10,18 +10,10 @@ config:
   security.nesting: "true"
   security.privileged: "true"
 devices:
-  aadisable:
-    path: /sys/module/nf_conntrack/parameters/hashsize
-    source: /sys/module/nf_conntrack/parameters/hashsize
-    type: disk
   aadisable2:
     path: /dev/kmsg
     source: /dev/kmsg
     type: unix-char
-  aadisable4:
-    path: /proc/sys/net/netfilter/nf_conntrack_max
-    source: /proc/sys/net/netfilter/nf_conntrack_max
-    type: disk
   dev-loop-control:
     major: "10"
     minor: "237"