Adds configurable containerd base dir during bootstrap / join (#817)

* Adds configurable containerd base dir during bootstrap / join

Currently, for the classic k8s snap, the default locations for the
containerd-related files are:

- /etc/containerd/
- /run/containerd/
- /var/lib/containerd/

These paths can conflict with other containerd installations on the host
(e.g. from docker), meaning that the k8s snap cannot be installed. In a
developer's usecase, we shouldn't require the other services to be
disabled, but we can allow the k8s snap's containerd to be installed in
a different location during bootstrap / node join.

Adds the --containerd-base-dir option for k8s bootstrap and join-cluster
commands. If not given, the snap's default paths will be used instead.
After the node was initialized, save the containerd base location in a
file, so we can properly reference it later / on restart.

k8s/lib.sh

@@ -99,6 +99,7 @@ k8s::remove::containerd() {
 
   # only remove containerd if the snap was already bootstrapped.
   # this is to prevent removing containerd when it is not installed by the snap.
+  # NOTE: do NOT include .containerd-base-dir! By default, it will contain "/".
   for file in "containerd-socket-path" "containerd-config-dir" "containerd-root-dir" "containerd-cni-bin-dir"; do
     if [ -f "$SNAP_COMMON/lock/$file" ]; then
       rm -rf $(cat "$SNAP_COMMON/lock/$file")

src/k8s/cmd/util/environ.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/canonical/k8s/pkg/snap"
@@ -32,10 +33,17 @@ func DefaultExecutionEnvironment() ExecutionEnvironment {
 	var s snap.Snap
 	switch os.Getenv("K8SD_RUNTIME_ENVIRONMENT") {
 	case "", "snap":
+		// If this node is already bootstrapped / joined, we should already know where the
+		// containerd base directory is. If not, leave the defaults.
+		containerdBaseDir := ""
+		if data, err := os.ReadFile(filepath.Join(os.Getenv("SNAP_COMMON"), "lock", snap.ContainerdBaseDir)); err == nil {
+			containerdBaseDir = strings.TrimSpace(string(data))
+		}
 		s = snap.NewSnap(snap.SnapOpts{
-			SnapDir:          os.Getenv("SNAP"),
-			SnapCommonDir:    os.Getenv("SNAP_COMMON"),
-			SnapInstanceName: os.Getenv("SNAP_INSTANCE_NAME"),
+			SnapDir:           os.Getenv("SNAP"),
+			SnapCommonDir:     os.Getenv("SNAP_COMMON"),
+			SnapInstanceName:  os.Getenv("SNAP_INSTANCE_NAME"),
+			ContainerdBaseDir: containerdBaseDir,
 		})
 	case "pebble":
 		s = snap.NewPebble(snap.PebbleOpts{

src/k8s/pkg/k8sd/api/cluster_bootstrap.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"path/filepath"
 	"time"
 
 	apiv1 "github.com/canonical/k8s-snap-api/api/v1"
@@ -40,6 +41,12 @@ func (e *Endpoints) postClusterBootstrap(_ state.State, r *http.Request) respons
 		return response.BadRequest(fmt.Errorf("cluster is already bootstrapped"))
 	}
 
+	// If not set, leave the default base dir location.
+	if req.Config.ContainerdBaseDir != "" {
+		// append k8s-containerd to the given base dir, so we don't flood it with our own folders.
+		e.provider.Snap().SetContainerdBaseDir(filepath.Join(req.Config.ContainerdBaseDir, "k8s-containerd"))
+	}
+
 	// NOTE(neoaggelos): microcluster adds an implicit 30 second timeout if no context deadline is set.
 	ctx, cancel := context.WithTimeout(r.Context(), time.Hour)
 	defer cancel()

src/k8s/pkg/k8sd/api/cluster_join.go

@@ -4,13 +4,15 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"path/filepath"
 	"time"
 
 	apiv1 "github.com/canonical/k8s-snap-api/api/v1"
 	"github.com/canonical/k8s/pkg/k8sd/types"
 	"github.com/canonical/k8s/pkg/utils"
 	"github.com/canonical/lxd/lxd/response"
 	"github.com/canonical/microcluster/v2/state"
+	"gopkg.in/yaml.v2"
 )
 
 func (e *Endpoints) postClusterJoin(s state.State, r *http.Request) response.Response {
@@ -33,6 +35,20 @@ func (e *Endpoints) postClusterJoin(s state.State, r *http.Request) response.Res
 		return NodeInUse(fmt.Errorf("node %q is part of the cluster", hostname))
 	}
 
+	joinConfig := struct {
+		// We only care about this field from the entire join config.
+		containerdBaseDir string `yaml:"containerd-base-dir,omitempty"`
+	}{}
+
+	if err := yaml.Unmarshal([]byte(req.Config), &joinConfig); err != nil {
+		return response.BadRequest(fmt.Errorf("failed to parse request config: %w", err))
+	}
+
+	if joinConfig.containerdBaseDir != "" {
+		// append k8s-containerd to the given base dir, so we don't flood it with our own folders.
+		e.provider.Snap().SetContainerdBaseDir(filepath.Join(joinConfig.containerdBaseDir, "k8s-containerd"))
+	}
+
 	config := map[string]string{}
 
 	// NOTE(neoaggelos): microcluster adds an implicit 30 second timeout if no context deadline is set.

src/k8s/pkg/k8sd/setup/containerd.go

@@ -173,6 +173,7 @@ func saveSnapContainerdPaths(s snap.Snap) error {
 		"containerd-config-dir":  s.ContainerdConfigDir(),
 		"containerd-root-dir":    s.ContainerdRootDir(),
 		"containerd-cni-bin-dir": s.CNIBinDir(),
+		snap.ContainerdBaseDir:   s.GetContainerdBaseDir(),
 	}
 
 	for filename, content := range m {

src/k8s/pkg/snap/interface.go

@@ -39,6 +39,8 @@ type Snap interface {
 	EtcdPKIDir() string          // /etc/kubernetes/pki/etcd
 	KubeletRootDir() string      // /var/lib/kubelet
 
+	SetContainerdBaseDir(baseDir string) // sets the containerd base directory.
+	GetContainerdBaseDir() string        // gets the containerd base directory.
 	ContainerdConfigDir() string         // classic confinement: /etc/containerd, strict confinement: /var/snap/k8s/common/etc/containerd
 	ContainerdExtraConfigDir() string    // classic confinement: /etc/containerd/conf.d, strict confinement: /var/snap/k8s/common/etc/containerd/conf.d
 	ContainerdRegistryConfigDir() string // classic confinement: /etc/containerd/hosts.d, strict confinement: /var/snap/k8s/common/etc/containerd/hosts.d

src/k8s/pkg/snap/mock/mock.go

@@ -30,6 +30,7 @@ type Mock struct {
 	ContainerdConfigDir         string
 	ContainerdExtraConfigDir    string
 	ContainerdRegistryConfigDir string
+	ContainerdBaseDir           string
 	ContainerdRootDir           string
 	ContainerdSocketDir         string
 	ContainerdSocketPath        string
@@ -131,6 +132,14 @@ func (s *Snap) Hostname() string {
 	return s.Mock.Hostname
 }
 
+func (s *Snap) SetContainerdBaseDir(baseDir string) {
+	s.Mock.ContainerdBaseDir = baseDir
+}
+
+func (s *Snap) GetContainerdBaseDir() string {
+	return s.Mock.ContainerdBaseDir
+}
+
 func (s *Snap) ContainerdConfigDir() string {
 	return s.Mock.ContainerdConfigDir
 }

src/k8s/pkg/snap/snap.go

@@ -24,6 +24,10 @@ import (
 	"k8s.io/cli-runtime/pkg/genericclioptions"
 )
 
+const (
+	ContainerdBaseDir = ".containerd-base-dir"
+)
+
 type SnapOpts struct {
 	SnapInstanceName  string
 	SnapDir           string
@@ -173,6 +177,14 @@ func (s *snap) Hostname() string {
 	return hostname
 }
 
+func (s *snap) SetContainerdBaseDir(baseDir string) {
+	s.containerdBaseDir = baseDir
+}
+
+func (s *snap) GetContainerdBaseDir() string {
+	return s.containerdBaseDir
+}
+
 func (s *snap) ContainerdConfigDir() string {
 	return filepath.Join(s.containerdBaseDir, "etc", "containerd")
 }
@@ -349,7 +361,8 @@ func (s *snap) PreInitChecks(ctx context.Context, config types.ClusterConfig, se
 	if _, err := os.Stat(s.ContainerdSocketDir()); err == nil {
 		return fmt.Errorf("The path '%s' required for the containerd socket already exists. "+
 			"This may mean that another service is already using that path, and it conflicts with the k8s snap. "+
-			"Please make sure that there is no other service installed that uses the same path, and remove the existing directory.", s.ContainerdSocketDir())
+			"Please make sure that there is no other service installed that uses the same path, and remove the existing directory."+
+			"(dev-only): You can change the default k8s containerd base path with the containerd-base-dir option in the bootstrap / join-cluster config file.", s.ContainerdSocketDir())
 	} else if !errors.Is(err, os.ErrNotExist) {
 		return fmt.Errorf("Encountered an error while checking '%s': %w", s.ContainerdSocketDir(), err)
 	}

tests/integration/templates/bootstrap-containerd-path.yaml

@@ -0,0 +1,2 @@
+# Contains the bootstrap configuration for the containerd-related paths test.
+containerd-base-dir: /home/ubuntu

tests/integration/tests/test_cleanup.py

@@ -2,19 +2,21 @@
 # Copyright 2024 Canonical, Ltd.
 #
 import logging
+import os
 from typing import List
 
 import pytest
-from test_util import harness, tags, util
+import yaml
+from test_util import config, harness, tags, util
 
 LOG = logging.getLogger(__name__)
 
 CONTAINERD_PATHS = [
     "/etc/containerd",
-    "/opt/cni/bin",
     "/run/containerd",
     "/var/lib/containerd",
 ]
+CNI_PATH = "/opt/cni/bin"
 
 
 @pytest.mark.node_count(1)
@@ -27,11 +29,65 @@ def test_node_cleanup(instances: List[harness.Instance], tmp_path):
     util.remove_k8s_snap(instance)
 
     # Check that the containerd-related folders are removed on snap removal.
+    all_paths = CONTAINERD_PATHS + [CNI_PATH]
     process = instance.exec(
-        ["ls", *CONTAINERD_PATHS], capture_output=True, text=True, check=False
+        ["ls", *all_paths], capture_output=True, text=True, check=False
     )
-    for path in CONTAINERD_PATHS:
+    for path in all_paths:
         assert f"cannot access '{path}': No such file or directory" in process.stderr
 
     util.setup_k8s_snap(instance, tmp_path)
     instance.exec(["k8s", "bootstrap"])
+
+
+@pytest.mark.node_count(2)
+@pytest.mark.disable_k8s_bootstrapping()
+@pytest.mark.tags(tags.NIGHTLY)
+def test_node_cleanup_new_containerd_path(instances: List[harness.Instance]):
+    main = instances[0]
+    joiner = instances[1]
+
+    containerd_path_bootstrap_config = (
+        config.MANIFESTS_DIR / "bootstrap-containerd-path.yaml"
+    ).read_text()
+
+    main.exec(
+        ["k8s", "bootstrap", "--file", "-"],
+        input=str.encode(containerd_path_bootstrap_config),
+    )
+
+    join_token = util.get_join_token(main, joiner)
+    joiner.exec(
+        ["k8s", "join-cluster", join_token, "--file", "-"],
+        input=str.encode(containerd_path_bootstrap_config),
+    )
+
+    boostrap_config = yaml.safe_load(containerd_path_bootstrap_config)
+    new_containerd_paths = [
+        os.path.join(boostrap_config["containerd-base-dir"], "k8s-containerd", p)
+        for p in CONTAINERD_PATHS
+    ]
+    for instance in instances:
+        # Check that the containerd-related folders are not in the default locations.
+        process = instance.exec(
+            ["ls", *CONTAINERD_PATHS], capture_output=True, text=True, check=False
+        )
+        for path in CONTAINERD_PATHS:
+            assert (
+                f"cannot access '{path}': No such file or directory" in process.stderr
+            )
+
+        # Check that the containerd-related folders are in the new locations.
+        # If one of them is missing, this should have a non-zero exit code.
+        instance.exec(["ls", *new_containerd_paths], check=True)
+
+    for instance in instances:
+        # Check that the containerd-related folders are not in the new locations after snap removal.
+        util.remove_k8s_snap(instance)
+        process = instance.exec(
+            ["ls", *new_containerd_paths], capture_output=True, text=True, check=False
+        )
+        for path in new_containerd_paths:
+            assert (
+                f"cannot access '{path}': No such file or directory" in process.stderr
+            )