diff --git a/build-scripts/build-component.sh b/build-scripts/build-component.sh index 0d9569ddb..149250eba 100755 --- a/build-scripts/build-component.sh +++ b/build-scripts/build-component.sh @@ -20,7 +20,7 @@ COMPONENT_BUILD_DIRECTORY="${BUILD_DIRECTORY}/${COMPONENT_NAME}" # cleanup git repository if we cannot git checkout to the build tag if [ -d "${COMPONENT_BUILD_DIRECTORY}" ]; then cd "${COMPONENT_BUILD_DIRECTORY}" - if ! git checkout "${GIT_TAG}"; then + if ! git reset --hard "${GIT_TAG}"; then cd "${BUILD_DIRECTORY}" rm -rf "${COMPONENT_BUILD_DIRECTORY}" fi diff --git a/build-scripts/components/cni/build.sh b/build-scripts/components/cni/build.sh index 353db2bf0..3ee70d770 100755 --- a/build-scripts/components/cni/build.sh +++ b/build-scripts/components/cni/build.sh @@ -2,7 +2,7 @@ VERSION="${2}" -INSTALL="${1}/opt/cni/bin" +INSTALL="${1}/bin" mkdir -p "${INSTALL}" # these would very tedious to apply with a patch @@ -15,7 +15,4 @@ export CGO_ENABLED=0 go build -o cni -ldflags "-s -w -extldflags -static -X github.com/containernetworking/plugins/pkg/utils/buildversion.BuildVersion=${VERSION}" ./cni.go -cp cni "${INSTALL}/" -for plugin in dhcp host-local static bridge host-device ipvlan loopback macvlan ptp vlan bandwidth firewall portmap sbr tuning vrf; do - ln -f -s ./cni "${INSTALL}/${plugin}" -done +cp cni "${INSTALL}/cni" diff --git a/k8s/components/components.yaml b/k8s/components/components.yaml deleted file mode 100644 index c7a042a8b..000000000 --- a/k8s/components/components.yaml +++ /dev/null @@ -1,24 +0,0 @@ -network: - release: "ck-network" - chart: "cilium-1.14.1.tgz" - namespace: "kube-system" -dns: - release: "ck-dns" - # TODO: [KU-373] We should fork the coredns chart for args changes - # and try to push it upstream - chart: "coredns-1.29.0" - namespace: "kube-system" -storage: - release: "ck-storage" - chart: "rawfile-csi-0.8.0.tgz" - namespace: "kube-system" -ingress: - parent: "network" -gateway: - release: "ck-gateway" - chart: "gateway-api-0.7.1.tgz" - namespace: "kube-system" -loadbalancer: - release: "ck-loadbalancer" - chart: "ck-loadbalancer" - namespace: "kube-system" diff --git a/k8s/lib.sh b/k8s/lib.sh index c9d0cc03c..35bdb698f 100755 --- a/k8s/lib.sh +++ b/k8s/lib.sh @@ -71,16 +71,6 @@ k8s::remove::network() { done } -# Run an openssl command -# Example: 'k8s::cmd::openssl genrsa 2048' -k8s::cmd::openssl() { - k8s::common::setup_env - - env \ - OPENSSL_CONF="${SNAP}/etc/ssl/openssl.cnf" \ - "${SNAP}/usr/bin/openssl" "${@}" -} - # Run a ctr command against the local containerd socket # Example: 'k8s::cmd::ctr image ls -q' k8s::cmd::ctr() { @@ -133,32 +123,6 @@ k8s::util::default_interface() { ip route show default | awk '{for(i=1; i component.crt' -k8s::util::pki::sign_cert() { - k8s::common::setup_env - - csr="$(cat)" - - # Parse SANs from the CSR and add them to the certificate extensions (if any) - extensions="" - alt_names="$(echo "$csr" | k8s::cmd::openssl req -text | grep "X509v3 Subject Alternative Name:" -A1 | tail -n 1 | sed 's,IP Address:,IP:,g')" - if test "x$alt_names" != "x"; then - extensions="subjectAltName = $alt_names" - fi - - # Sign certificate and print to stdout - echo "$csr" | k8s::cmd::openssl x509 -req -sha256 -CA /etc/kubernetes/pki/ca.crt -CAkey /etc/kubernetes/pki/ca.key -CAcreateserial -days 3650 -extfile <(echo "${extensions}") "${@}" -} - # Execute a "$SNAP/bin/$service" with arguments from "$SNAP_DATA/args/$service" # Example: 'k8s::common::execute_service kubelet' k8s::common::execute_service() { @@ -225,165 +149,18 @@ k8s::common::execute_service() { k8s::common::setup_env # Source arguments and substitute environment variables. Will fail if we cannot read the file. - declare -a args="($(cat "${SNAP_DATA}/args/${service_name}"))" + declare -a args="($(cat "${SNAP_COMMON}/args/${service_name}"))" set -xe exec "${SNAP}/bin/${service_name}" "${args[@]}" } -# Initialize a single-node k8s-dqlite -k8s::init::k8s_dqlite() { - k8s::common::setup_env - - k8s::cmd::openssl req -x509 -newkey rsa:4096 -sha256 -days 3650 -nodes \ - -keyout /var/lib/k8s-dqlite/cluster.key -out /var/lib/k8s-dqlite/cluster.crt \ - -config "$SNAP/k8s/csr/csr.conf" -extensions v3_ext \ - -subj /CN=k8s \ - -addext "subjectAltName = DNS:k8s-dqlite, IP:127.0.0.1" - echo 'Address: "127.0.0.1:2380"' > /var/lib/k8s-dqlite/init.yaml - - mkdir -p "$SNAP_DATA/args" - cp "$SNAP/k8s/args/k8s-dqlite" "$SNAP_DATA/args/k8s-dqlite" -} - # Initialize a single-node k8sd cluster k8s::init::k8sd() { k8s::common::setup_env - mkdir -p "$SNAP_DATA/args" - cp "$SNAP/k8s/args/k8sd" "$SNAP_DATA/args/k8sd" -} - -# Initialize containerd for the local node -k8s::init::containerd() { - k8s::common::setup_env - - mkdir -p "$SNAP_DATA/args" - cp "$SNAP/k8s/args/containerd" "$SNAP_DATA/args/containerd" - cp "$SNAP/k8s/config/containerd/config.toml" "$SNAP_COMMON/etc/containerd/config.toml" - cp "$SNAP/opt/cni/bin/"* /opt/cni/bin/ -} - -# Initialize Kubernetes PKI CA (self-signed) -# Example: 'k8s::init::ca' -k8s::init::ca() { - k8s::common::setup_env - - mkdir -p /etc/kubernetes/pki - - for key in serviceaccount ca front-proxy-ca; do - k8s::util::pki::generate_key "/etc/kubernetes/pki/${key}.key" - done - - # Generate Kubernetes CA - k8s::cmd::openssl req -x509 -new -sha256 -nodes -days 3650 -key /etc/kubernetes/pki/ca.key -subj "/CN=kubernetes-ca" -out /etc/kubernetes/pki/ca.crt - # Generate Front Proxy CA - k8s::cmd::openssl req -x509 -new -sha256 -nodes -days 3650 -key /etc/kubernetes/pki/front-proxy-ca.key -subj "/CN=kubernetes-front-proxy-ca" -out /etc/kubernetes/pki/front-proxy-ca.crt -} - -# Initialize Kuberentes server and client certificates, using our own self-signed CA. -# Example: 'k8s::init::pki' -k8s::init::pki() { - k8s::common::setup_env - - # Generate kube-apiserver certificate - # TODO(neoaggelos): add IP addresses of machine, add extra SANs from user configuration - k8s::util::pki::generate_csr "/CN=kube-apiserver" /etc/kubernetes/pki/apiserver.key -addext "$(echo "subjectAltName = - DNS: localhost, - DNS: kubernetes, - DNS: kubernetes.default, - DNS: kubernetes.default.svc, - DNS: kubernetes.default.svc.cluster, - DNS: $(k8s::cmd::hostname), - - IP: 127.0.0.1, - IP: 10.152.183.1, - IP: $(k8s::util::default_ip) - " | tr '\n' ' ')" | k8s::util::pki::sign_cert > /etc/kubernetes/pki/apiserver.crt - - # Generate front-proxy-client certificate (signed by front-proxy-ca) - k8s::util::pki::generate_csr /CN=front-proxy-client /etc/kubernetes/pki/front-proxy-client.key -config "$SNAP/k8s/csr/csr.conf" | - k8s::util::pki::sign_cert -extensions v3_ext -extfile "$SNAP/k8s/csr/csr.conf" -CA "/etc/kubernetes/pki/front-proxy-ca.crt" -CAkey "/etc/kubernetes/pki/front-proxy-ca.key" \ - > /etc/kubernetes/pki/front-proxy-client.crt - - # Generate kubelet certificates - # TODO(neoaggelos): add IP addresses of machine - k8s::util::pki::generate_csr "/CN=system:node:$(k8s::cmd::hostname)/O=system:nodes" /etc/kubernetes/pki/kubelet.key -addext "$(echo "subjectAltName = - DNS: $(k8s::cmd::hostname), - IP: 127.0.0.1, - IP: $(k8s::util::default_ip) - " | tr '\n' ' ')" | k8s::util::pki::sign_cert > /etc/kubernetes/pki/kubelet.crt - - # Generate the rest of the client certificates - k8s::util::pki::generate_csr /CN=kubernetes-admin/O=system:masters /etc/kubernetes/pki/admin.key | k8s::util::pki::sign_cert > /etc/kubernetes/pki/admin.crt - k8s::util::pki::generate_csr /CN=system:kube-proxy /etc/kubernetes/pki/proxy.key | k8s::util::pki::sign_cert > /etc/kubernetes/pki/proxy.crt - k8s::util::pki::generate_csr /CN=system:kube-scheduler /etc/kubernetes/pki/scheduler.key | k8s::util::pki::sign_cert > /etc/kubernetes/pki/scheduler.crt - k8s::util::pki::generate_csr /CN=system:kube-controller-manager /etc/kubernetes/pki/controller-manager.key | k8s::util::pki::sign_cert > /etc/kubernetes/pki/controller-manager.crt - k8s::util::pki::generate_csr /CN=kube-apiserver-kubelet-client/O=system:masters /etc/kubernetes/pki/apiserver-kubelet-client.key | k8s::util::pki::sign_cert > /etc/kubernetes/pki/apiserver-kubelet-client.crt -} - -k8s::init::kubeconfigs() { - k8s::util::generate_x509_kubeconfig /etc/kubernetes/pki/admin.crt /etc/kubernetes/pki/admin.key /etc/kubernetes/pki/ca.crt > /etc/kubernetes/admin.conf - k8s::util::generate_x509_kubeconfig /etc/kubernetes/pki/kubelet.crt /etc/kubernetes/pki/kubelet.key /etc/kubernetes/pki/ca.crt > /etc/kubernetes/kubelet.conf - k8s::util::generate_x509_kubeconfig /etc/kubernetes/pki/proxy.crt /etc/kubernetes/pki/proxy.key /etc/kubernetes/pki/ca.crt > /etc/kubernetes/proxy.conf - k8s::util::generate_x509_kubeconfig /etc/kubernetes/pki/controller-manager.crt /etc/kubernetes/pki/controller-manager.key /etc/kubernetes/pki/ca.crt > /etc/kubernetes/controller-manager.conf - k8s::util::generate_x509_kubeconfig /etc/kubernetes/pki/scheduler.crt /etc/kubernetes/pki/scheduler.key /etc/kubernetes/pki/ca.crt > /etc/kubernetes/scheduler.conf -} - -# Generate a kubeconfig file that uses x509 certificates for authentication. -# Example: 'k8s::util::generate_x509_kubeconfig /etc/kubernetes/pki/admin.crt /etc/kubernetes/pki/admin.key /etc/kubernetes/pki/ca.crt 127.0.0.1 6443 > /etc/kubernetes/admin.conf' -k8s::util::generate_x509_kubeconfig() { - k8s::common::setup_env - - cert_data="$(base64 -w 0 < "$1")" - key_data="$(base64 -w 0 < "$2")" - ca_data="$(base64 -w 0 < "$3")" - - # optional arguments (apiserver IP and port) - apiserver="${4:-127.0.0.1}" - apiserver_port="$(cat "$SNAP_DATA/args/kube-apiserver" | grep -- --secure-port | tr '=' ' ' | cut -f2 -d' ')" - port="${5:-$apiserver_port}" - - cat "$SNAP/k8s/config/kubeconfig" | - sed 's/CADATA/'"${ca_data}"'/g' | - sed 's/CERTDATA/'"${cert_data}"'/g' | - sed 's/KEYDATA/'"${key_data}"'/g' | - sed 's/APISERVER/'"${apiserver}"'/g' | - sed 's/PORT/'"${port}"'/g' -} - -# Configure default arguments for Kubernetes services -# Example: 'k8s::init::kubernetes' -k8s::init::kubernetes() { - k8s::common::setup_env - - mkdir -p "$SNAP_DATA/args" - cp "$SNAP/k8s/args/kubelet" "$SNAP_DATA/args/kubelet" - cp "$SNAP/k8s/args/kube-apiserver" "$SNAP_DATA/args/kube-apiserver" - cp "$SNAP/k8s/args/kube-proxy" "$SNAP_DATA/args/kube-proxy" - cp "$SNAP/k8s/args/kube-scheduler" "$SNAP_DATA/args/kube-scheduler" - cp "$SNAP/k8s/args/kube-controller-manager" "$SNAP_DATA/args/kube-controller-manager" -} - -# Configure permissions for important cluster config files -# Example: 'k8s::init::permissions' -k8s::init::permissions() { - k8s::common::setup_env - - chmod go-rxw -R "$SNAP_DATA/args" "$SNAP_COMMON/opt" "$SNAP_COMMON/etc" "$SNAP_COMMON/var/lib" "$SNAP_COMMON/var/log" -} - -# Initialize all services to run a single-node cluster -# Example: 'k8s::init' -k8s::init() { - k8s::init::containerd - k8s::init::k8s_dqlite - k8s::init::k8sd - k8s::init::ca - k8s::init::pki - k8s::init::kubernetes - k8s::init::kubeconfigs - k8s::init::permissions + mkdir -m 0700 -p "$SNAP_COMMON/args" + cp "$SNAP/k8s/args/k8sd" "$SNAP_COMMON/args/k8sd" } # Ensure /var/lib/kubelet is a shared mount diff --git a/src/k8s/api/v1/worker.go b/src/k8s/api/v1/worker.go index b4962c8b3..7e9abe683 100644 --- a/src/k8s/api/v1/worker.go +++ b/src/k8s/api/v1/worker.go @@ -5,6 +5,8 @@ package v1 type WorkerNodeInfoRequest struct { // Hostname is the name of the worker node. Hostname string `json:"hostname"` + // Address is the address of the worker node. + Address string `json:"address"` } // WorkerNodeInfoResponse is used to return a worker node token. @@ -17,12 +19,16 @@ type WorkerNodeInfoResponse struct { KubeletToken string `json:"kubeletToken"` // KubeProxyToken is the token to use for kube-proxy. KubeProxyToken string `json:"kubeProxyToken"` - // ClusterCIDR is the configured cluster CIDR. - ClusterCIDR string `json:"clusterCIDR"` + // PodCIDR is the configured CIDR for pods in the cluster. + PodCIDR string `json:"podCIDR"` // ClusterDNS is the DNS server address of the cluster. ClusterDNS string `json:"clusterDNS,omitempty"` // ClusterDomain is the DNS domain of the cluster. ClusterDomain string `json:"clusterDomain,omitempty"` // CloudProvider is the cloud provider used in the cluster. CloudProvider string `json:"cloudProvider,omitempty"` + // KubeletCert is the certificate to use for kubelet TLS. It will be empty if the cluster is not using self-signed certificates. + KubeletCert string `json:"kubeletCrt,omitempty"` + // KubeletKey is the private key to use for kubelet TLS. It will be empty if the cluster is not using self-signed certificates. + KubeletKey string `json:"kubeletKey,omitempty"` } diff --git a/src/k8s/cmd/k8s/k8s_cluster.go b/src/k8s/cmd/k8s/k8s_cluster.go index 448c85332..0dd332108 100644 --- a/src/k8s/cmd/k8s/k8s_cluster.go +++ b/src/k8s/cmd/k8s/k8s_cluster.go @@ -2,7 +2,8 @@ package k8s import ( "os" - "path" + + "github.com/canonical/k8s/pkg/snap" ) var ( @@ -12,7 +13,7 @@ var ( ) func init() { - rootCmd.PersistentFlags().StringVar(&clusterCmdOpts.stateDir, "state-dir", path.Join(os.Getenv("SNAP_COMMON"), "/var/lib/k8sd/state"), "Directory with the dqlite datastore") + rootCmd.PersistentFlags().StringVar(&clusterCmdOpts.stateDir, "state-dir", snap.NewSnap(os.Getenv("SNAP"), os.Getenv("SNAP_COMMON")).K8sdStateDir(), "Directory with the dqlite datastore") // By default, the state dir is set to a fixed directory in the snap. // This shouldn't be overwritten by the user. diff --git a/src/k8s/cmd/k8sd/k8sd.go b/src/k8s/cmd/k8sd/k8sd.go index 7300f9290..44dbeb2f6 100644 --- a/src/k8s/cmd/k8sd/k8sd.go +++ b/src/k8s/cmd/k8sd/k8sd.go @@ -3,10 +3,10 @@ package k8sd import ( "fmt" "os" - "path" "github.com/canonical/k8s/pkg/config" "github.com/canonical/k8s/pkg/k8sd/app" + "github.com/canonical/k8s/pkg/snap" "github.com/spf13/cobra" ) @@ -22,11 +22,13 @@ var ( Use: "k8sd", Short: "Canonical Kubernetes orchestrator and clustering daemon", RunE: func(cmd *cobra.Command, args []string) error { + snap := snap.NewSnap(os.Getenv("SNAP"), os.Getenv("SNAP_COMMON")) app, err := app.New(cmd.Context(), app.Config{ Debug: rootCmdOpts.logDebug, Verbose: rootCmdOpts.logVerbose, StateDir: rootCmdOpts.stateDir, ListenPort: rootCmdOpts.port, + Snap: snap, }) if err != nil { return fmt.Errorf("failed to initialize k8sd: %w", err) @@ -44,5 +46,5 @@ func init() { rootCmd.PersistentFlags().BoolVarP(&rootCmdOpts.logDebug, "debug", "d", false, "Show all debug messages") rootCmd.PersistentFlags().BoolVarP(&rootCmdOpts.logVerbose, "verbose", "v", true, "Show all information messages") rootCmd.PersistentFlags().UintVar(&rootCmdOpts.port, "port", config.DefaultPort, "Port on which the REST API is exposed") - rootCmd.PersistentFlags().StringVar(&rootCmdOpts.stateDir, "state-dir", path.Join(os.Getenv("SNAP_COMMON"), "/var/lib/k8sd/state"), "Directory with the dqlite datastore") + rootCmd.PersistentFlags().StringVar(&rootCmdOpts.stateDir, "state-dir", "", "Directory with the dqlite datastore") } diff --git a/src/k8s/cmd/k8sd/k8sd_sql.go b/src/k8s/cmd/k8sd/k8sd_sql.go index adc2fcb07..829e12360 100644 --- a/src/k8s/cmd/k8sd/k8sd_sql.go +++ b/src/k8s/cmd/k8sd/k8sd_sql.go @@ -2,8 +2,10 @@ package k8sd import ( "fmt" + "os" "github.com/canonical/k8s/pkg/k8sd/app" + "github.com/canonical/k8s/pkg/snap" "github.com/spf13/cobra" ) @@ -13,14 +15,15 @@ var ( Short: "Execute an SQL query against the daemon", Hidden: true, RunE: func(cmd *cobra.Command, args []string) error { - if len(args) != 1 { return fmt.Errorf("invalid query") } + snap := snap.NewSnap(os.Getenv("SNAP"), os.Getenv("SNAP_COMMON")) cluster, err := app.New(cmd.Context(), app.Config{ Debug: rootCmdOpts.logDebug, Verbose: rootCmdOpts.logVerbose, StateDir: rootCmdOpts.stateDir, + Snap: snap, }) if err != nil { return fmt.Errorf("failed to create k8sd app: %w", err) diff --git a/src/k8s/go.mod b/src/k8s/go.mod index c53bd81ea..a86e2efbf 100644 --- a/src/k8s/go.mod +++ b/src/k8s/go.mod @@ -15,7 +15,6 @@ require ( github.com/prometheus/client_golang v1.17.0 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.0 - github.com/spf13/viper v1.18.0 go.etcd.io/etcd/api/v3 v3.5.10 go.etcd.io/etcd/client/pkg/v3 v3.5.10 go.etcd.io/etcd/client/v3 v3.5.10 @@ -96,7 +95,6 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect github.com/huandu/xstrings v1.4.0 // indirect github.com/imdario/mergo v0.3.13 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -113,7 +111,6 @@ require ( github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect github.com/lib/pq v1.10.9 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect - github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect @@ -121,7 +118,6 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/moby/locker v1.0.1 // indirect github.com/moby/spdystream v0.2.0 // indirect @@ -135,7 +131,6 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0-rc5 // indirect github.com/pborman/uuid v1.2.1 // indirect - github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pkg/sftp v1.13.6 // indirect github.com/pkg/xattr v0.4.9 // indirect @@ -148,16 +143,11 @@ require ( github.com/rogpeppe/fastuuid v1.2.0 // indirect github.com/rubenv/sql-migrate v1.5.2 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/sagikazarmark/locafero v0.4.0 // indirect - github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect github.com/soheilhy/cmux v0.1.5 // indirect - github.com/sourcegraph/conc v0.3.0 // indirect - github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/testify v1.8.4 // indirect - github.com/subosito/gotenv v1.6.0 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect @@ -182,7 +172,6 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.21.0 // indirect golang.org/x/crypto v0.16.0 // indirect - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/net v0.19.0 // indirect golang.org/x/oauth2 v0.15.0 // indirect golang.org/x/term v0.15.0 // indirect @@ -196,7 +185,6 @@ require ( gopkg.in/errgo.v1 v1.0.1 // indirect gopkg.in/httprequest.v1 v1.2.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/macaroon.v2 v2.1.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect diff --git a/src/k8s/go.sum b/src/k8s/go.sum index 46959a236..4127edf69 100644 --- a/src/k8s/go.sum +++ b/src/k8s/go.sum @@ -380,7 +380,6 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= @@ -452,8 +451,6 @@ github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= -github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/markbates/errx v1.1.0 h1:QDFeR+UP95dO12JgW+tgi2UVfo0V8YBHiUIOaeBPiEI= @@ -496,8 +493,6 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= @@ -541,8 +536,6 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= -github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/peterh/liner v1.2.1/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= @@ -600,10 +593,6 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= -github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= -github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= -github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= @@ -620,11 +609,7 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= -github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= -github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= -github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= @@ -635,8 +620,6 @@ github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= -github.com/spf13/viper v1.18.0 h1:pN6W1ub/G4OfnM+NR9p7xP9R6TltLUzp5JG9yZD3Qg0= -github.com/spf13/viper v1.18.0/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -654,8 +637,6 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= -github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -768,8 +749,6 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1146,8 +1125,6 @@ gopkg.in/httprequest.v1 v1.2.1/go.mod h1:x2Otw96yda5+8+6ZeWwHIJTFkEHWP/qP8pJOzqE gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/macaroon.v2 v2.1.0 h1:HZcsjBCzq9t0eBPMKqTN/uSN6JOm78ZJ2INbqcBQOUI= gopkg.in/macaroon.v2 v2.1.0/go.mod h1:OUb+TQP/OP0WOerC2Jp/3CwhIKyIa9kQjuc7H24e6/o= gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw= diff --git a/src/k8s/pkg/component/component.go b/src/k8s/pkg/component/component.go index 9db0113ba..5df83e8f4 100644 --- a/src/k8s/pkg/component/component.go +++ b/src/k8s/pkg/component/component.go @@ -5,9 +5,9 @@ import ( "os" "sort" + "github.com/canonical/k8s/pkg/k8sd/types" "github.com/canonical/k8s/pkg/snap" "github.com/sirupsen/logrus" - "github.com/spf13/viper" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/cli" @@ -16,18 +16,9 @@ import ( // defaultHelmConfigProvider implements the HelmConfigInitializer interface type defaultHelmConfigProvider struct{} -// componentDefinition defines each component metadata. -type componentDefinition struct { - ParentComponent string `mapstructure:"parent"` - ReleaseName string `mapstructure:"release"` - Chart string `mapstructure:"chart"` - Namespace string `mapstructure:"namespace"` -} - // helmClient implements the ComponentManager interface type helmClient struct { - config map[string]componentDefinition - snap snap.Snap + components map[string]types.Component initializer HelmConfigProvider } @@ -70,29 +61,15 @@ func NewHelmClient(snap snap.Snap, initializer HelmConfigProvider) (*helmClient, initializer = &defaultHelmConfigProvider{} } - viper.SetConfigName("components") - viper.SetConfigType("yaml") - viper.AddConfigPath(snap.Path("k8s/components")) - - if err := viper.ReadInConfig(); err != nil { - return nil, err - } - - config := make(map[string]componentDefinition) - if err := viper.Unmarshal(&config); err != nil { - return nil, err - } - return &helmClient{ - config: config, - snap: snap, + components: snap.Components(), initializer: initializer, }, nil } // Enable enables a specified component. func (h *helmClient) Enable(name string, values map[string]any) error { - component, ok := h.config[name] + component, ok := h.components[name] if !ok { return fmt.Errorf("invalid component %s", name) } @@ -115,7 +92,7 @@ func (h *helmClient) Enable(name string, values map[string]any) error { return nil } - chart, err := loader.Load(h.snap.Path("k8s/components/charts", component.Chart)) + chart, err := loader.Load(component.ManifestPath) if err != nil { return fmt.Errorf("failed to load component manifest: %w", err) } @@ -164,11 +141,11 @@ func (h *helmClient) List() ([]Component, error) { return nil, fmt.Errorf("failed to list components: %w", err) } - allComponents := make([]Component, 0, len(h.config)) + allComponents := make([]Component, 0, len(h.components)) componentsMap := make(map[string]int) // Loop through components and populate allComponents and componentsMap - for name, component := range h.config { + for name, component := range h.components { index := len(componentsMap) allComponents = append(allComponents, Component{Name: name}) @@ -191,7 +168,7 @@ func (h *helmClient) List() ([]Component, error) { // Disable disables a specified component. func (h *helmClient) Disable(name string) error { - component, ok := h.config[name] + component, ok := h.components[name] if !ok { return fmt.Errorf("invalid component %s", name) } @@ -221,7 +198,7 @@ func (h *helmClient) Disable(name string) error { // Refresh refreshes a specified component. func (h *helmClient) Refresh(name string, values map[string]any) error { - component, ok := h.config[name] + component, ok := h.components[name] if !ok { return fmt.Errorf("invalid component %s", name) } @@ -235,7 +212,7 @@ func (h *helmClient) Refresh(name string, values map[string]any) error { upgrade.Namespace = component.Namespace upgrade.ReuseValues = true - chart, err := loader.Load(h.snap.Path("k8s/components/charts", component.Chart)) + chart, err := loader.Load(component.ManifestPath) if err != nil { return fmt.Errorf("failed to load component manifest: %w", err) } diff --git a/src/k8s/pkg/component/component_test.go b/src/k8s/pkg/component/component_test.go index ff5c239b4..482054f1a 100644 --- a/src/k8s/pkg/component/component_test.go +++ b/src/k8s/pkg/component/component_test.go @@ -7,6 +7,7 @@ import ( "testing" componentmock "github.com/canonical/k8s/pkg/component/mock" + "github.com/canonical/k8s/pkg/k8sd/types" snapmock "github.com/canonical/k8s/pkg/snap/mock" . "github.com/onsi/gomega" "helm.sh/helm/v3/pkg/action" @@ -97,21 +98,6 @@ func mustMakeMeSomeReleases(store *storage.Storage, t *testing.T) (all []*releas var componentsNone = `` -var components = ` -one: - release: "whiskas-1" - chart: "chunky-tuna-1.14.1.tgz" - namespace: "default" -two: - release: "whiskas-2" - chart: "tuna-1.29.0.tgz" - namespace: "default" -three: - release: "whiskas-3" - chart: "chunky-1.29.0.tgz" - namespace: "default" -` - func mustCreateTemporaryTestDirectory(t *testing.T) string { // Create a temporary test directory to mock the snap // @@ -137,7 +123,7 @@ func mustAddConfigToTestDir(t *testing.T, path string, data string) { } } -func mustCreateNewHelmClient(t *testing.T, components string) (*helmClient, string, *action.Configuration) { +func mustCreateNewHelmClient(t *testing.T, components map[string]types.Component) (*helmClient, string, *action.Configuration) { // Create a mock actionConfig for testing mockActionConfig := actionConfigFixture(t) // Create a mock HelmClient with the desired behavior for testing @@ -146,12 +132,11 @@ func mustCreateNewHelmClient(t *testing.T, components string) (*helmClient, stri // create test directory to use for the snap mock tempDir := mustCreateTemporaryTestDirectory(t) - // Create a file and add some configs - mustAddConfigToTestDir(t, filepath.Join(tempDir, "k8s", "components", "components.yaml"), components) - // Create mock snap snap := &snapmock.Snap{ - PathPrefix: tempDir, + Mock: snapmock.Mock{ + Components: components, + }, } //Create a mock ComponentManager with the mock HelmClient @@ -166,7 +151,7 @@ func mustCreateNewHelmClient(t *testing.T, components string) (*helmClient, stri func TestListEmptyComponents(t *testing.T) { g := NewWithT(t) // Create a mock ComponentManager with no components - mockHelmClient, tempDir, _ := mustCreateNewHelmClient(t, componentsNone) + mockHelmClient, tempDir, _ := mustCreateNewHelmClient(t, nil) defer os.RemoveAll(tempDir) // Call the List function with the mock HelmClient @@ -181,7 +166,24 @@ func TestListComponentsWithReleases(t *testing.T) { // Create a mock ComponentManager with the mock HelmClient // This mock uses components.yaml for the snap mock components - mockHelmClient, tempDir, mockActionConfig := mustCreateNewHelmClient(t, components) + mockHelmClient, tempDir, mockActionConfig := mustCreateNewHelmClient(t, map[string]types.Component{ + "one": { + ReleaseName: "whiskas-1", + Namespace: "default", + ManifestPath: "chunky-tuna-1.14.1.tgz", + }, + "two": { + ReleaseName: "whiskas-2", + Namespace: "default", + ManifestPath: "tuna-1.29.0.tgz", + }, + "three": { + ReleaseName: "whiskas-3", + Namespace: "default", + ManifestPath: "chunky-1.29.0.tgz", + }, + }) + defer os.RemoveAll(tempDir) // Create releases in the mock actionConfig @@ -192,9 +194,9 @@ func TestListComponentsWithReleases(t *testing.T) { components, err := mockHelmClient.List() g.Expect(err).To(BeNil()) - g.Expect(components).To(HaveLen(3)) - - g.Expect(components[0]).To(Equal(Component{Name: "one", Status: true})) - g.Expect(components[2]).To(Equal(Component{Name: "two", Status: true})) - g.Expect(components[1]).To(Equal(Component{Name: "three", Status: true})) + g.Expect(components).To(Equal([]Component{ + {Name: "one", Status: true}, + {Name: "three", Status: true}, + {Name: "two", Status: true}, + })) } diff --git a/src/k8s/pkg/component/dns.go b/src/k8s/pkg/component/dns.go index 920ab3842..c416582a3 100644 --- a/src/k8s/pkg/component/dns.go +++ b/src/k8s/pkg/component/dns.go @@ -7,13 +7,16 @@ import ( "time" "github.com/canonical/k8s/pkg/snap" + snaputil "github.com/canonical/k8s/pkg/snap/util" "github.com/canonical/k8s/pkg/utils/k8s" ) -func EnableDNSComponent(s snap.Snap, clusterDomain, serviceIP string, upstreamNameservers []string) error { +// EnableDNSComponent enables DNS on the cluster. +// On success, it returns the IP of the DNS service and the cluster domain. +func EnableDNSComponent(s snap.Snap, clusterDomain, serviceIP string, upstreamNameservers []string) (string, string, error) { manager, err := NewHelmClient(s, nil) if err != nil { - return fmt.Errorf("failed to get component manager: %w", err) + return "", "", fmt.Errorf("failed to get component manager: %w", err) } upstreamNameserver := "/etc/resolv.conf" @@ -69,12 +72,12 @@ func EnableDNSComponent(s snap.Snap, clusterDomain, serviceIP string, upstreamNa err = manager.Enable("dns", values) if err != nil { - return fmt.Errorf("failed to enable dns component: %w", err) + return "", "", fmt.Errorf("failed to enable dns component: %w", err) } - client, err := k8s.NewClient() + client, err := k8s.NewClient(s) if err != nil { - return fmt.Errorf("failed to create kubernetes client: %w", err) + return "", "", fmt.Errorf("failed to create kubernetes client: %w", err) } ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) @@ -82,18 +85,15 @@ func EnableDNSComponent(s snap.Snap, clusterDomain, serviceIP string, upstreamNa dnsIP, err := k8s.GetServiceClusterIP(ctx, client, "coredns", "kube-system") if err != nil { - return fmt.Errorf("failed to get dns service: %w", err) - } - - // TODO: Use database.SetClusterConfig() to store ClusterDNS and ClusterDomain - kubeletArgs := []map[string]string{ - {"--cluster-dns": dnsIP}, - {"--cluster-domain": clusterDomain}, + return "", "", fmt.Errorf("failed to get dns service: %w", err) } - changed, err := snap.UpdateServiceArguments(s, "kubelet", kubeletArgs, []string{}) + changed, err := snaputil.UpdateServiceArguments(s, "kubelet", map[string]string{ + "--cluster-dns": dnsIP, + "--cluster-domain": clusterDomain, + }, nil) if err != nil { - return fmt.Errorf("failed to update 'kubelet' arguments: %w", err) + return "", "", fmt.Errorf("failed to update kubelet arguments: %w", err) } if changed { @@ -102,11 +102,10 @@ func EnableDNSComponent(s snap.Snap, clusterDomain, serviceIP string, upstreamNa err = s.RestartService(ctx, "kubelet") if err != nil { - return fmt.Errorf("failed to restart service 'kubelet': %w", err) + return "", "", fmt.Errorf("failed to restart kubelet to apply new dns configuration: %w", err) } - } - return nil + return dnsIP, clusterDomain, nil } func DisableDNSComponent(s snap.Snap) error { @@ -120,15 +119,9 @@ func DisableDNSComponent(s snap.Snap) error { return fmt.Errorf("failed to disable dns component: %w", err) } - kubeletArgs := []map[string]string{ - {"--cluster-domain": "cluster.local"}, - } - - removeArgs := []string{"--cluster-dns"} - - changed, err := snap.UpdateServiceArguments(s, "kubelet", kubeletArgs, removeArgs) + changed, err := snaputil.UpdateServiceArguments(s, "kubelet", map[string]string{"--cluster-domain": "cluster.local"}, nil) if err != nil { - return fmt.Errorf("failed to update 'kubelet' arguments: %w", err) + return fmt.Errorf("failed to update kubelet arguments: %w", err) } if changed { @@ -139,7 +132,6 @@ func DisableDNSComponent(s snap.Snap) error { if err != nil { return fmt.Errorf("failed to restart service 'kubelet': %w", err) } - } return nil diff --git a/src/k8s/pkg/component/gateway.go b/src/k8s/pkg/component/gateway.go index 1e7827bdf..e5dfc1db6 100644 --- a/src/k8s/pkg/component/gateway.go +++ b/src/k8s/pkg/component/gateway.go @@ -33,7 +33,7 @@ func EnableGatewayComponent(s snap.Snap) error { return fmt.Errorf("failed to enable gateway component: %w", err) } - client, err := k8s.NewClient() + client, err := k8s.NewClient(s) if err != nil { return fmt.Errorf("failed to create kubernetes client: %w", err) } diff --git a/src/k8s/pkg/component/ingress.go b/src/k8s/pkg/component/ingress.go index cecfe1111..8fba94a27 100644 --- a/src/k8s/pkg/component/ingress.go +++ b/src/k8s/pkg/component/ingress.go @@ -30,7 +30,7 @@ func EnableIngressComponent(s snap.Snap, defaultTLSSecret string, enableProxyPro return fmt.Errorf("failed to enable ingress component: %w", err) } - client, err := k8s.NewClient() + client, err := k8s.NewClient(s) if err != nil { return fmt.Errorf("failed to create kubernetes client: %w", err) } diff --git a/src/k8s/pkg/component/loadbalancer.go b/src/k8s/pkg/component/loadbalancer.go index 1b0c9f746..fc341b839 100644 --- a/src/k8s/pkg/component/loadbalancer.go +++ b/src/k8s/pkg/component/loadbalancer.go @@ -68,7 +68,7 @@ func EnableLoadBalancerComponent(s snap.Snap, cidrs []string, l2Enabled bool, l2 return fmt.Errorf("failed to enable loadbalancer component: %w", err) } - client, err := k8s.NewClient() + client, err := k8s.NewClient(s) if err != nil { return fmt.Errorf("failed to create kubernetes client: %w", err) } diff --git a/src/k8s/pkg/component/network.go b/src/k8s/pkg/component/network.go index 5b10cf33e..38ede1e66 100644 --- a/src/k8s/pkg/component/network.go +++ b/src/k8s/pkg/component/network.go @@ -9,15 +9,13 @@ import ( "github.com/canonical/k8s/pkg/utils" ) -func EnableNetworkComponent(s snap.Snap) error { +func EnableNetworkComponent(s snap.Snap, podCIDR string) error { manager, err := NewHelmClient(s, nil) if err != nil { return fmt.Errorf("failed to get component manager: %w", err) } - // TODO: the cluster cidr should be configurable through a common interface - clusterCIDRStr := snap.GetServiceArgument(s, "kube-proxy", "--cluster-cidr") - clusterCIDRs := strings.Split(clusterCIDRStr, ",") + clusterCIDRs := strings.Split(podCIDR, ",") if v := len(clusterCIDRs); v != 1 && v != 2 { return fmt.Errorf("invalid kube-proxy --cluster-cidr value: %v", clusterCIDRs) } @@ -70,7 +68,7 @@ func EnableNetworkComponent(s snap.Snap) error { }, } - if s.IsStrict() { + if s.Strict() { bpfMnt, err := utils.GetMountPath("bpf") if err != nil { return fmt.Errorf("failed to get bpf mount path: %w", err) diff --git a/src/k8s/pkg/k8s/client/client.go b/src/k8s/pkg/k8s/client/client.go index 60c62a782..dcd8c094f 100644 --- a/src/k8s/pkg/k8s/client/client.go +++ b/src/k8s/pkg/k8s/client/client.go @@ -3,19 +3,23 @@ package client import ( "context" "fmt" + "os" + "github.com/canonical/k8s/pkg/snap" "github.com/canonical/microcluster/client" "github.com/canonical/microcluster/microcluster" ) // ClusterOpts contains options for cluster queries. type ClusterOpts struct { - // StateDir is the directory that contains the cluster state (for local clients). - StateDir string // Verbose enables info level logging. Verbose bool // Debug enables trace level logging. Debug bool + // Snap is the snap instance. + Snap snap.Snap + // StateDir is the directory that contains the cluster state (for local clients). + StateDir string } // Client interacts with the k8s REST-API via unix-socket or HTTPS @@ -23,12 +27,21 @@ type Client struct { opts ClusterOpts m *microcluster.MicroCluster mc *client.Client + snap snap.Snap } // NewClient returns a client to interact with the k8s REST-API // On a cluster node it will return a client connected to the unix-socket // elsewhere it returns a HTTPS client that expects the certificates to be located at ClusterOpts.StateDir func NewClient(ctx context.Context, opts ClusterOpts) (*Client, error) { + // TODO: pass snap through opts instead, do not create here. + if opts.Snap == nil { + opts.Snap = snap.NewSnap(os.Getenv("SNAP"), os.Getenv("SNAP_COMMON")) + } + stateDir := opts.StateDir + if stateDir == "" { + stateDir = opts.Snap.K8sdStateDir() + } m, err := microcluster.App(ctx, microcluster.Args{ Debug: opts.Debug, StateDir: opts.StateDir, diff --git a/src/k8s/pkg/k8s/client/cluster.go b/src/k8s/pkg/k8s/client/cluster.go index 5977e5e91..0a5b141a8 100644 --- a/src/k8s/pkg/k8s/client/cluster.go +++ b/src/k8s/pkg/k8s/client/cluster.go @@ -46,7 +46,8 @@ func (c *Client) Bootstrap(ctx context.Context, bootstrapConfig apiv1.BootstrapC } if err := c.m.NewCluster(hostname, addrPort, config, time.Second*30); err != nil { // TODO(neoaggelos): print message that bootstrap failed, and that we are cleaning up - c.CleanupNode(ctx, hostname) + fmt.Fprintln(os.Stderr, "Failed with error:", err) + c.CleanupNode(ctx, c.opts.Snap, hostname) return apiv1.ClusterMember{}, fmt.Errorf("failed to bootstrap new cluster: %w", err) } diff --git a/src/k8s/pkg/k8s/client/cluster_node.go b/src/k8s/pkg/k8s/client/cluster_node.go index 2f2b667f8..3e670cb11 100644 --- a/src/k8s/pkg/k8s/client/cluster_node.go +++ b/src/k8s/pkg/k8s/client/cluster_node.go @@ -7,6 +7,7 @@ import ( apiv1 "github.com/canonical/k8s/api/v1" "github.com/canonical/k8s/pkg/snap" + snaputil "github.com/canonical/k8s/pkg/snap/util" "github.com/canonical/k8s/pkg/utils/control" "github.com/canonical/lxd/shared/api" ) @@ -24,9 +25,9 @@ func (c *Client) JoinCluster(ctx context.Context, name string, address string, t var response apiv1.JoinClusterResponse err := c.mc.Query(ctx, "POST", api.NewURL().Path("k8sd", "cluster", "join"), request, &response) if err != nil { - fmt.Fprintln(os.Stderr, "failed to join node - cleaning up now") - c.CleanupNode(ctx, name) - return fmt.Errorf("failed to query endpoint POST /k8sd/cluster/join: %w", err) + fmt.Fprintln(os.Stderr, "Failed with error:", err) + + c.CleanupNode(ctx, c.opts.Snap, name) } c.WaitForDqliteNodeToBeReady(ctx, name) @@ -65,7 +66,7 @@ func (c *Client) WaitForDqliteNodeToBeReady(ctx context.Context, nodeName string // CleanupNode resets the nodes configuration and cluster state. // The cleanup will happen on a best-effort base. Any error that occurs will be ignored. -func (c *Client) CleanupNode(ctx context.Context, nodeName string) { +func (c *Client) CleanupNode(ctx context.Context, snap snap.Snap, nodeName string) { // For self-removal, microcluster expects the dqlite node to not be in pending state. c.WaitForDqliteNodeToBeReady(ctx, nodeName) @@ -78,5 +79,6 @@ func (c *Client) CleanupNode(ctx context.Context, nodeName string) { // joining another cluster. c.ResetNode(ctx, nodeName, true) - snap.StopControlPlaneServices(ctx, snap.NewDefaultSnap()) + // TODO(neoaggelos): reenable after we know how to pass a snap here + snaputil.StopControlPlaneServices(ctx, snap) } diff --git a/src/k8s/pkg/k8s/setup/certificates.go b/src/k8s/pkg/k8s/setup/certificates.go deleted file mode 100644 index 437940e20..000000000 --- a/src/k8s/pkg/k8s/setup/certificates.go +++ /dev/null @@ -1,42 +0,0 @@ -package setup - -import ( - "fmt" - - "github.com/canonical/k8s/pkg/utils/cert" -) - -// InitCertificates sets up the CAs and the necessary server certificates that is used by Kubernetes. -// An initial CertificateAuthority can be provided. If not, a self-signed one will be generated. -func InitCertificates(ca *cert.CertKeyPair) (*cert.CertificateManager, error) { - certMan, err := cert.NewCertificateManager() - if err != nil { - return nil, fmt.Errorf("failed to create certificate manager: %w", err) - } - - if ca != nil { - certMan.CA = ca - } else { - err = certMan.GenerateCA() - if err != nil { - return nil, fmt.Errorf("failed to generate certificate authority: %w", err) - } - } - - err = certMan.GenerateFrontProxyCA() - if err != nil { - return nil, fmt.Errorf("failed to generate front proxy certificate authority: %w", err) - } - - err = certMan.GenerateServerCerts() - if err != nil { - return nil, fmt.Errorf("failed to generate server certificates: %w", err) - } - - err = certMan.GenerateServiceAccountKey() - if err != nil { - return nil, fmt.Errorf("failed to generate service account key: %w", err) - } - - return certMan, nil -} diff --git a/src/k8s/pkg/k8s/setup/containerd.go b/src/k8s/pkg/k8s/setup/containerd.go deleted file mode 100644 index f28505974..000000000 --- a/src/k8s/pkg/k8s/setup/containerd.go +++ /dev/null @@ -1,29 +0,0 @@ -package setup - -import ( - "fmt" - - "github.com/canonical/k8s/pkg/snap" - "github.com/canonical/k8s/pkg/utils" -) - -// InitContainerd handles the setup of containerd. -// - Copies required files and binaries needed by Containerd to the correct paths. -func InitContainerd(snap snap.Snap) error { - err := utils.CopyFile(snap.Path("k8s/config/containerd/config.toml"), snap.CommonPath("/etc/containerd/config.toml")) - if err != nil { - return fmt.Errorf("failed to copy containerd config: %w", err) - } - - err = utils.CopyDirectory(snap.Path("opt/cni/bin/"), "/opt/cni/bin/") - if err != nil { - return fmt.Errorf("failed to copy cni/bin: %w", err) - } - - err = utils.ChmodRecursive("/opt/cni/bin/", 0700) - if err != nil { - return fmt.Errorf("failed to adjust permissions of /opt/cni/bin: %w", err) - } - - return nil -} diff --git a/src/k8s/pkg/k8s/setup/folders.go b/src/k8s/pkg/k8s/setup/folders.go deleted file mode 100644 index 61f6a177c..000000000 --- a/src/k8s/pkg/k8s/setup/folders.go +++ /dev/null @@ -1,35 +0,0 @@ -package setup - -import ( - "fmt" - "os" - - "github.com/canonical/k8s/pkg/utils/cert" -) - -// InitFolders creates the necessary folders for service arguments and certificates. -func InitFolders(argsDir string) error { - err := os.MkdirAll(argsDir, os.ModePerm) - if err != nil { - return fmt.Errorf("failed to create arguments directory: %w", err) - } - - err = os.MkdirAll(cert.KubePkiPath, os.ModePerm) - if err != nil { - return fmt.Errorf("failed to create pki directory: %w", err) - } - - if err := os.MkdirAll("/opt/cni/bin", 0700); err != nil { - return fmt.Errorf("failed to create cni bin dir: %w", err) - } - - if err := os.MkdirAll("/etc/cni/net.d", 0700); err != nil { - return fmt.Errorf("failed to create cni conf dir: %w", err) - } - - // TODO(neoaggelos): don't use a hardcoded path here - if err := os.MkdirAll("/var/snap/k8s/common/etc/containerd", 0700); err != nil { - return fmt.Errorf("failed to create cni bin dir: %w", err) - } - return nil -} diff --git a/src/k8s/pkg/k8s/setup/k8s_dqlite.go b/src/k8s/pkg/k8s/setup/k8s_dqlite.go index e99590284..532634b1c 100644 --- a/src/k8s/pkg/k8s/setup/k8s_dqlite.go +++ b/src/k8s/pkg/k8s/setup/k8s_dqlite.go @@ -3,70 +3,14 @@ package setup import ( "context" "fmt" - "os" - "path/filepath" "github.com/canonical/k8s/pkg/snap" "github.com/canonical/k8s/pkg/utils" - "github.com/canonical/k8s/pkg/utils/cert" "github.com/canonical/k8s/pkg/utils/dqlite" "github.com/canonical/microcluster/state" - "gopkg.in/yaml.v2" ) -// JoinK8sDqliteCluster joins a node to an existing k8s-dqlite cluster. It: -// -// - retrieves k8s-dqlite certificates and address from cluster node (k8sd is already joined at this point so we can access the certificates) -// - stores new certificates in k8s-dqlite cluster directory -// - writes k8s-dqlite init file with the cluster node information -// - starts k8s-dqlite -func JoinK8sDqliteCluster(ctx context.Context, state *state.State, snap snap.Snap) error { - clusterConfig, err := utils.GetClusterConfig(ctx, state) - if err != nil { - return fmt.Errorf("failed to get cluster config: %w", err) - } - - k8sDqliteCertPair, err := cert.NewCertKeyPairFromPEM([]byte(clusterConfig.Certificates.K8sDqliteCert), []byte(clusterConfig.Certificates.K8sDqliteKey)) - if err != nil { - return fmt.Errorf("failed to create k8s-dqlite cert from pem: %w", err) - } - - if err := k8sDqliteCertPair.SaveCertificate(snap.CommonPath(cert.K8sDqlitePkiPath, "cluster.crt")); err != nil { - return fmt.Errorf("failed to write k8s-dqlite cert: %w", err) - } - if err := k8sDqliteCertPair.SavePrivateKey(snap.CommonPath(cert.K8sDqlitePkiPath, "cluster.key")); err != nil { - return fmt.Errorf("failed to write k8s-dqlite key: %w", err) - } - - leader, err := state.Leader() - if err != nil { - return fmt.Errorf("failed to get dqlite leader: %w", err) - } - - members, err := leader.GetClusterMembers(ctx) - if err != nil { - return fmt.Errorf("failed to get dqlite members: %w", err) - } - clusterAddrs := make([]string, len(members)) - - for _, member := range members { - clusterAddrs = append(clusterAddrs, fmt.Sprintf("%s:%d", member.Address.Addr(), clusterConfig.K8sDqlite.Port)) - } - - initFile := K8sDqliteInit{ - Cluster: clusterAddrs, - } - if err := WriteClusterInitFile(initFile); err != nil { - return fmt.Errorf("failed to update cluster info.yaml file: %w", err) - } - - if err := snap.StartService(ctx, "k8s-dqlite"); err != nil { - return fmt.Errorf("failed to stop k8s-dqlite: %w", err) - } - - return nil -} - +// TODO(neoaggelos): this is not part of the cluster setup. func LeaveK8sDqliteCluster(ctx context.Context, snap snap.Snap, state *state.State) error { clusterConfig, err := utils.GetClusterConfig(ctx, state) if err != nil { @@ -85,28 +29,6 @@ func LeaveK8sDqliteCluster(ctx context.Context, snap snap.Snap, state *state.Sta return fmt.Errorf("failed to leave cluster: %w", err) } - return utils.RunCommand(ctx, snap.Path("k8s/wrappers/commands/dqlite"), "k8s", fmt.Sprintf(".remove %s", address)) -} - -// K8sDqliteInit represents the yaml file structure of the dqlite `init.yaml` file. -type K8sDqliteInit struct { - ID uint64 `yaml:"ID,omitempty"` - Address string `yaml:"Address,omitempty"` - Role int `yaml:"Role,omitempty"` - Cluster []string `yaml:"Cluster,omitempty"` -} - -// WriteClusterInitFile writes an `init.yaml` file to the k8s-dqlite directory -// that contains the informations to join an existing cluster (e.g. members addresses) -// and is picked up by k8s-dqlite on startup. -func WriteClusterInitFile(init K8sDqliteInit) error { - marshaled, err := yaml.Marshal(&init) - if err != nil { - return fmt.Errorf("failed to marshal cluster init data: %w", err) - } - - if err := os.WriteFile(filepath.Join("/var/snap/k8s/common", cert.K8sDqlitePkiPath, "init.yaml"), []byte(marshaled), 0644); err != nil { - return fmt.Errorf("failed to write init.yaml to %s: %w", cert.K8sDqlitePkiPath, err) - } - return nil + // TODO: do not use the dqlite shell to remove the node. + return utils.RunCommand(ctx, "/snap/k8s/current/k8s/wrappers/commands/dqlite", "k8s", fmt.Sprintf(".remove %s", address)) } diff --git a/src/k8s/pkg/k8s/setup/kube_apiserver.go b/src/k8s/pkg/k8s/setup/kube_apiserver.go deleted file mode 100644 index 397540155..000000000 --- a/src/k8s/pkg/k8s/setup/kube_apiserver.go +++ /dev/null @@ -1,30 +0,0 @@ -package setup - -import ( - "fmt" - - "github.com/canonical/k8s/pkg/config" - "github.com/canonical/k8s/pkg/utils" -) - -// InitKubeApiserver handles the setup of kube-apiserver. -// - Sets up the token webhook authentication. -func InitKubeApiserver(apiServerTokenHookPathTemplate string) error { - defaultIp, err := utils.GetDefaultIP() - if err != nil { - return fmt.Errorf("failed to get default ip: %w", err) - } - - utils.TemplateAndSave(apiServerTokenHookPathTemplate, - struct { - WebhookIp string - WebhookPort int - }{ - WebhookIp: defaultIp.String(), - WebhookPort: config.DefaultPort, - }, - "/etc/kubernetes/apiserver-token-hook.conf", - ) - - return nil -} diff --git a/src/k8s/pkg/k8s/setup/kubeconfigs.go b/src/k8s/pkg/k8s/setup/kubeconfigs.go deleted file mode 100644 index 14aee8bea..000000000 --- a/src/k8s/pkg/k8s/setup/kubeconfigs.go +++ /dev/null @@ -1,104 +0,0 @@ -package setup - -import ( - "context" - "encoding/base64" - "fmt" - "os" - - apiImpl "github.com/canonical/k8s/pkg/k8sd/api/impl" - "github.com/canonical/k8s/pkg/snap" - "github.com/canonical/k8s/pkg/utils" - "github.com/canonical/k8s/pkg/utils/cert" - "github.com/canonical/microcluster/state" -) - -// InitKubeconfigs generates the kubeconfig files that services use to communicate with the apiserver. -func InitKubeconfigs(ctx context.Context, state *state.State, ca *cert.CertKeyPair, hostOverwrite *string, portOverwrite *int) error { - hostname, err := os.Hostname() - if err != nil { - return fmt.Errorf("failed to get hostname: %w", err) - } - - type KubeconfigArgs struct { - username string - groups []string - path string - } - - configs := []KubeconfigArgs{ - { - username: "kubernetes-admin", - groups: []string{"system:masters"}, - path: "/etc/kubernetes/admin.conf", - }, - { - username: "system:kube-controller-manager", - groups: []string{}, - path: "/etc/kubernetes/controller-manager.conf", - }, - { - username: "system:kube-proxy", - groups: []string{}, - path: "/etc/kubernetes/proxy.conf", - }, - { - username: "system:kube-scheduler", - groups: []string{}, - path: "/etc/kubernetes/scheduler.conf", - }, - { - username: fmt.Sprintf("system:node:%s", hostname), - groups: []string{"system:nodes"}, - path: "/etc/kubernetes/kubelet.conf", - }, - } - - for _, config := range configs { - token, err := apiImpl.GetOrCreateAuthToken(ctx, state, config.username, config.groups) - if err != nil { - return fmt.Errorf("could not generate auth token for %s: %w", config.username, err) - } - - err = renderKubeconfig(snap.SnapFromContext(state.Context), token, ca.CertPem, config.path, hostOverwrite, portOverwrite) - if err != nil { - return fmt.Errorf("failed to generate kubeconfig for %s: %w", config.username, err) - } - } - - return nil -} - -// renderKubeconfig creates a kubeconfig file with the given token and CA data. -func renderKubeconfig(snap snap.Snap, token string, caCertPem []byte, path string, hostOverwrite *string, portOverwrite *int) error { - port := apiServerPort(snap, portOverwrite) - return utils.TemplateAndSave(snap.Path("k8s/config/kubeconfig-with-token.tmpl"), - struct { - CaData string - ApiServerIp string - ApiServerPort string - Token string - }{ - CaData: base64.StdEncoding.EncodeToString(caCertPem), - ApiServerIp: apiServerHost(hostOverwrite), - ApiServerPort: port, - Token: token, - }, - path, - ) -} - -func apiServerHost(hostOverwrite *string) string { - if hostOverwrite != nil { - return *hostOverwrite - } - return "127.0.0.1" -} - -func apiServerPort(s snap.Snap, portOverwrite *int) string { - if portOverwrite != nil { - return fmt.Sprintf("%d", *portOverwrite) - } else { - return snap.GetServiceArgument(s, "kube-apiserver", "--secure-port") - } -} diff --git a/src/k8s/pkg/k8s/setup/kubeconfigs_test.go b/src/k8s/pkg/k8s/setup/kubeconfigs_test.go deleted file mode 100644 index 0b80d0367..000000000 --- a/src/k8s/pkg/k8s/setup/kubeconfigs_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package setup - -import ( - "fmt" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/canonical/k8s/pkg/snap" - "github.com/canonical/k8s/pkg/utils" - . "github.com/onsi/gomega" -) - -func TestRenderKubeconfig(t *testing.T) { - wd, err := os.Getwd() - if err != nil { - t.Fatal(err) - } - // Use the repository dir as the snap dir so that the template path is resolved - snapDir := filepath.Join(wd, "../../../../../") - mockSnap := snap.NewSnap( - snapDir, - "", - "", - ) - - testCases := []struct { - name string - hostOverwrite string - portOverwrite int - expectedKubeconfig string - }{ - { - name: "withOverwrites", - hostOverwrite: "192.168.12.3", - portOverwrite: 6000, - expectedKubeconfig: `apiVersion: v1 -clusters: -- cluster: - certificate-authority-data: bW9ja1BlbQ== - server: https://{{ .ApiServerIp }}:{{ .ApiServerPort }} - name: k8s -contexts: -- context: - cluster: k8s - user: k8s-user - name: k8s -current-context: k8s -kind: Config -users: -- name: k8s-user - user: - token: token -`, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - g := NewWithT(t) - file := filepath.Join(t.TempDir(), tc.name) - err := renderKubeconfig(mockSnap, "token", []byte("mockPem"), file, &tc.hostOverwrite, &tc.portOverwrite) - g.Expect(err).NotTo(HaveOccurred()) - content, err := utils.ReadFile(file) - g.Expect(err).NotTo(HaveOccurred()) - - expectedKubeconfig := strings.ReplaceAll(tc.expectedKubeconfig, "{{ .ApiServerIp }}", tc.hostOverwrite) - expectedKubeconfig = strings.ReplaceAll(expectedKubeconfig, "{{ .ApiServerPort }}", fmt.Sprintf("%d", tc.portOverwrite)) - g.Expect(content).To(Equal(expectedKubeconfig)) - }) - } -} diff --git a/src/k8s/pkg/k8s/setup/permissions.go b/src/k8s/pkg/k8s/setup/permissions.go deleted file mode 100644 index 341866c74..000000000 --- a/src/k8s/pkg/k8s/setup/permissions.go +++ /dev/null @@ -1,28 +0,0 @@ -package setup - -import ( - "context" - "fmt" - - "github.com/canonical/k8s/pkg/snap" - "github.com/canonical/k8s/pkg/utils" -) - -// InitPermissions makes sure(sets up) the permissions of paths utilized by the snap are correct. -func InitPermissions(ctx context.Context, snap snap.Snap) error { - // Shelling out since go doesn't support symbolic mode definitions. - err := utils.RunCommand(ctx, - "chmod", "go-rxw", "-R", - snap.DataPath("args"), - snap.CommonPath("etc"), - snap.CommonPath("var/lib"), - "/opt/cni/bin", - "/etc/kubernetes", - "/etc/cni/net.d", - ) - if err != nil { - return fmt.Errorf("failed to change folder permissions: %w", err) - } - - return nil -} diff --git a/src/k8s/pkg/k8s/setup/serviceargs.go b/src/k8s/pkg/k8s/setup/serviceargs.go deleted file mode 100644 index 7d34adaf5..000000000 --- a/src/k8s/pkg/k8s/setup/serviceargs.go +++ /dev/null @@ -1,35 +0,0 @@ -package setup - -import ( - "fmt" - - "github.com/canonical/k8s/pkg/snap" - "github.com/canonical/k8s/pkg/utils" -) - -var k8sServices = []string{"containerd", "k8s-dqlite", "kube-apiserver", "kube-controller-manager", "kube-proxy", "kube-scheduler", "kubelet"} - -// InitServiceArgs handles the setup of services arguments. -// - For each service, copies the default arguments files from the snap under $SNAP_DATA/args and apply any overwrites -// - Note that the `k8sd` service is already configured in the snap install hook and thus not included here -func InitServiceArgs(snap snap.Snap, extraArgs map[string]map[string]string) error { - for _, service := range k8sServices { - serviceArgs, err := utils.ParseArgumentFile(snap.Path("k8s/args", service)) - if err != nil { - return fmt.Errorf("failed to parse argument file for %s: %w", service, err) - } - - // Apply overwrites for each service - if args, exists := extraArgs[service]; exists { - for argument, value := range args { - serviceArgs[argument] = value - } - } - - if err := utils.SerializeArgumentFile(serviceArgs, snap.DataPath("args", service)); err != nil { - return fmt.Errorf("failed to write arguments file for %s: %w", service, err) - } - } - - return nil -} diff --git a/src/k8s/pkg/k8s/setup/serviceargs_test.go b/src/k8s/pkg/k8s/setup/serviceargs_test.go deleted file mode 100644 index 452ce846f..000000000 --- a/src/k8s/pkg/k8s/setup/serviceargs_test.go +++ /dev/null @@ -1,120 +0,0 @@ -package setup - -import ( - "os" - "path/filepath" - "testing" - - "github.com/canonical/k8s/pkg/snap" - "github.com/canonical/k8s/pkg/utils" - . "github.com/onsi/gomega" -) - -func TestInitServiceArgs(t *testing.T) { - snapDir := t.TempDir() - snapDataDir := t.TempDir() - snapArgsDir := filepath.Join(snapDir, "k8s/args") - dataArgsDir := filepath.Join(snapDataDir, "args") - - // Replace the snap instance with the temporary directory for testing - mockSnap := snap.NewSnap( - snapDir, - snapDataDir, - "", - ) - - testCases := []struct { - name string - initialFileContents map[string]string - overwrites map[string]map[string]string - expectedFileContents map[string]string - }{ - { - name: "joinOverwrites", - initialFileContents: map[string]string{ - "kube-apiserver": `--kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key ---kubelet-preferred-address-types=InternalIP,Hostname,InternalDNS,ExternalDNS,ExternalIP ---secure-port=6443 ---service-cluster-ip-range=10.152.183.0/24 -`, - }, - overwrites: map[string]map[string]string{ - "kube-apiserver": { - "--secure-port": "6000", - }, - }, - expectedFileContents: map[string]string{ - "kube-apiserver": `--kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key ---kubelet-preferred-address-types=InternalIP,Hostname,InternalDNS,ExternalDNS,ExternalIP ---secure-port=6000 ---service-cluster-ip-range=10.152.183.0/24 -`, - }, - }, - { - name: "emptyOverwrites", - initialFileContents: map[string]string{ - "kube-apiserver": `--authorization-mode=Node,RBAC ---client-ca-file=/etc/kubernetes/pki/ca.crt ---service-account-key-file=/etc/kubernetes/pki/serviceaccount.key ---service-cluster-ip-range=10.152.183.0/24 ---tls-cert-file=/etc/kubernetes/pki/apiserver.crt ---tls-private-key-file=/etc/kubernetes/pki/apiserver.key -`, - }, - overwrites: map[string]map[string]string{}, - expectedFileContents: map[string]string{ - "kube-apiserver": `--authorization-mode=Node,RBAC ---client-ca-file=/etc/kubernetes/pki/ca.crt ---service-account-key-file=/etc/kubernetes/pki/serviceaccount.key ---service-cluster-ip-range=10.152.183.0/24 ---tls-cert-file=/etc/kubernetes/pki/apiserver.crt ---tls-private-key-file=/etc/kubernetes/pki/apiserver.key -`, - }, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - g := NewWithT(t) - mustSetupArgsDirectoriesAndFiles(t, snapArgsDir, dataArgsDir, tc.initialFileContents) - - err := InitServiceArgs(mockSnap, tc.overwrites) - g.Expect(err).NotTo(HaveOccurred()) - - // Verify the content of the argument files in the temporary directory - verifyArgumentFileContent(t, dataArgsDir, tc.expectedFileContents) - }) - } -} - -func mustSetupArgsDirectoriesAndFiles(t *testing.T, snapArgsDir string, dataArgsDir string, fileContents map[string]string) { - g := NewWithT(t) - err := os.MkdirAll(snapArgsDir, 0755) - g.Expect(err).To(BeNil()) - - err = os.MkdirAll(dataArgsDir, 0755) - g.Expect(err).To(BeNil()) - - for _, service := range k8sServices { - content, exists := fileContents[service] - if !exists { - content = "" - } - err := os.WriteFile(filepath.Join(snapArgsDir, service), []byte(content), 0755) - g.Expect(err).To(BeNil()) - } -} - -func verifyArgumentFileContent(t *testing.T, dataArgsDir string, expected map[string]string) { - g := NewWithT(t) - for service, expectedContent := range expected { - filePath := filepath.Join(dataArgsDir, service) - - content, err := utils.ReadFile(filePath) - g.Expect(err).NotTo(HaveOccurred()) - - g.Expect(content).To(Equal(expectedContent)) - } -} diff --git a/src/k8s/pkg/k8s/setup/worker.go b/src/k8s/pkg/k8s/setup/worker.go deleted file mode 100644 index 5367579e8..000000000 --- a/src/k8s/pkg/k8s/setup/worker.go +++ /dev/null @@ -1,79 +0,0 @@ -package setup - -// TODO(neoaggelos): there is currently lots of duplicate code in this package, but it lets us move fast -// for lack of an easy way to unit test things. - -import ( - "fmt" - "os" - - "github.com/canonical/k8s/pkg/proxy" - "github.com/canonical/k8s/pkg/snap" - "github.com/canonical/k8s/pkg/utils" -) - -// initServiceArgs configures service arguments on a node. -// initServiceArgs uses default values from $SNAP/k8s/args/$service -// initServiceArgs updates arguments on updateArgs (override if arg exists, append otherwise). -// initServiceArgs removes arguments from deleteArgs. -// initServiceArgs writes the resulting arguments file at $SNAP_DATA/args/$service. -// TODO(neoaggelos): this currently duplicates logic from other helpers, we need to return to this. -func initServiceArgs(snap snap.Snap, service string, updateArgs map[string]string, deleteArgs []string) error { - args, err := utils.ParseArgumentFile(snap.Path("k8s", "args", service)) - if err != nil { - return fmt.Errorf("failed to parse default arguments for %s: %w", "kubelet", err) - } - for key, value := range updateArgs { - args[key] = value - } - for _, key := range deleteArgs { - delete(args, key) - } - - if err := utils.SerializeArgumentFile(args, snap.DataPath("args", service)); err != nil { - return fmt.Errorf("failed to write arguments file for kubelet: %w", err) - } - return nil -} - -// InitKubeletArgs configures kubelet on the node. -func InitKubeletArgs(snap snap.Snap, extraArgs map[string]string, deleteArgs []string) error { - return initServiceArgs(snap, "kubelet", extraArgs, deleteArgs) -} - -// InitKubeProxyArgs configures kube-proxy on the node. -func InitKubeProxyArgs(snap snap.Snap, extraArgs map[string]string, deleteArgs []string) error { - return initServiceArgs(snap, "kube-proxy", extraArgs, deleteArgs) -} - -// RenderKubeletKubeconfig renders the kubeconfig file for kubelet. -func RenderKubeletKubeconfig(snap snap.Snap, token string, caPEM string) error { - ip := "127.0.0.1" - port := 6443 - return renderKubeconfig(snap, token, []byte(caPEM), "/etc/kubernetes/kubelet.conf", &ip, &port) -} - -// RenderKubeProxyKubeconfig renders the kubeconfig file for kube-proxy. -func RenderKubeProxyKubeconfig(snap snap.Snap, token string, caPEM string) error { - ip := "127.0.0.1" - port := 6443 - return renderKubeconfig(snap, token, []byte(caPEM), "/etc/kubernetes/proxy.conf", &ip, &port) -} - -// WriteCA writes the CA certificate of the cluster. -func WriteCA(snap snap.Snap, crt string) error { - return os.WriteFile("/etc/kubernetes/pki/ca.crt", []byte(crt), 0600) -} - -func InitAPIServerProxy(snap snap.Snap, servers []string) error { - if err := proxy.WriteEndpointsConfig(servers, "/etc/kubernetes/k8s-apiserver-proxy.json"); err != nil { - return fmt.Errorf("failed to write proxy configuration file: %w", err) - } - - return initServiceArgs(snap, "k8s-apiserver-proxy", nil, nil) -} - -// InitContainerdArgs configures kube-proxy on the node. -func InitContainerdArgs(snap snap.Snap, extraArgs map[string]string, deleteArgs []string) error { - return initServiceArgs(snap, "containerd", extraArgs, deleteArgs) -} diff --git a/src/k8s/pkg/k8sd/api/component.go b/src/k8s/pkg/k8sd/api/component.go index ad32ef143..6f17d58d5 100644 --- a/src/k8s/pkg/k8sd/api/component.go +++ b/src/k8s/pkg/k8sd/api/component.go @@ -1,6 +1,8 @@ package api import ( + "context" + "database/sql" "encoding/json" "fmt" "net/http" @@ -9,7 +11,10 @@ import ( "github.com/canonical/k8s/pkg/component" "github.com/canonical/k8s/pkg/k8sd/api/impl" + "github.com/canonical/k8s/pkg/k8sd/database" + "github.com/canonical/k8s/pkg/k8sd/types" "github.com/canonical/k8s/pkg/snap" + "github.com/canonical/k8s/pkg/utils" "github.com/canonical/lxd/lxd/response" "github.com/canonical/microcluster/state" ) @@ -38,9 +43,25 @@ func putDNSComponent(s *state.State, r *http.Request) response.Response { switch req.Status { case api.ComponentEnable: - if err := component.EnableDNSComponent(snap, req.Config.ClusterDomain, req.Config.ServiceIP, req.Config.UpstreamNameservers); err != nil { + dnsIP, clusterDomain, err := component.EnableDNSComponent(snap, req.Config.ClusterDomain, req.Config.ServiceIP, req.Config.UpstreamNameservers) + if err != nil { return response.InternalError(fmt.Errorf("failed to enable dns: %w", err)) } + + if err := s.Database.Transaction(s.Context, func(ctx context.Context, tx *sql.Tx) error { + if err := database.SetClusterConfig(ctx, tx, types.ClusterConfig{ + Kubelet: types.Kubelet{ + ClusterDNS: dnsIP, + ClusterDomain: clusterDomain, + }, + }); err != nil { + return fmt.Errorf("failed to update cluster configuration for dns=%s domain=%s: %w", dnsIP, clusterDomain, err) + } + return nil + }); err != nil { + return response.InternalError(fmt.Errorf("database transaction to update cluster configuration failed: %w", err)) + } + case api.ComponentDisable: if err := component.DisableDNSComponent(snap); err != nil { return response.InternalError(fmt.Errorf("failed to disable dns: %w", err)) @@ -62,7 +83,11 @@ func putNetworkComponent(s *state.State, r *http.Request) response.Response { switch req.Status { case api.ComponentEnable: - if err := component.EnableNetworkComponent(snap); err != nil { + cfg, err := utils.GetClusterConfig(s.Context, s) + if err != nil { + return response.InternalError(fmt.Errorf("failed to retrieve pod cidr: %w", err)) + } + if err := component.EnableNetworkComponent(snap, cfg.Network.PodCIDR); err != nil { return response.InternalError(fmt.Errorf("failed to enable network: %w", err)) } case api.ComponentDisable: diff --git a/src/k8s/pkg/k8sd/api/impl/k8sd.go b/src/k8s/pkg/k8sd/api/impl/k8sd.go index 2fa6afe1b..5d821e2f9 100644 --- a/src/k8s/pkg/k8sd/api/impl/k8sd.go +++ b/src/k8s/pkg/k8sd/api/impl/k8sd.go @@ -15,7 +15,7 @@ import ( func GetClusterStatus(ctx context.Context, s *state.State) (apiv1.ClusterStatus, error) { snap := snap.SnapFromContext(s.Context) - k8sClient, err := k8s.NewClient() + k8sClient, err := k8s.NewClient(snap) if err != nil { return apiv1.ClusterStatus{}, fmt.Errorf("failed to create k8s client: %w", err) } diff --git a/src/k8s/pkg/k8sd/api/worker.go b/src/k8s/pkg/k8sd/api/worker.go index e9ed438d9..2340204fd 100644 --- a/src/k8s/pkg/k8sd/api/worker.go +++ b/src/k8s/pkg/k8sd/api/worker.go @@ -5,10 +5,13 @@ import ( "database/sql" "encoding/json" "fmt" + "net" "net/http" apiv1 "github.com/canonical/k8s/api/v1" "github.com/canonical/k8s/pkg/k8sd/database" + "github.com/canonical/k8s/pkg/k8sd/pki" + "github.com/canonical/k8s/pkg/snap" "github.com/canonical/k8s/pkg/utils" "github.com/canonical/k8s/pkg/utils/k8s" "github.com/canonical/lxd/lxd/response" @@ -44,13 +47,26 @@ func postWorkerInfo(s *state.State, r *http.Request) response.Response { if nodeName == "" { return response.BadRequest(fmt.Errorf("node name cannot be empty")) } + nodeIP := net.ParseIP(req.Address) + if nodeIP == nil { + return response.BadRequest(fmt.Errorf("failed to parse node IP address %s", req.Address)) + } - clusterConfig, err := utils.GetClusterConfig(s.Context, s) + cfg, err := utils.GetClusterConfig(s.Context, s) if err != nil { return response.InternalError(fmt.Errorf("failed to get cluster config: %w", err)) } - client, err := k8s.NewClient() + certificates := pki.NewControlPlanePKI(pki.ControlPlanePKIOpts{Years: 10}) + certificates.CACert = cfg.Certificates.CACert + certificates.CAKey = cfg.Certificates.CAKey + workerCertificates, err := certificates.CompleteWorkerNodePKI(nodeName, nodeIP, 2048) + if err != nil { + return response.InternalError(fmt.Errorf("failed to generate worker PKI: %w", err)) + } + + snap := snap.SnapFromContext(s.Context) + client, err := k8s.NewClient(snap) if err != nil { return response.InternalError(fmt.Errorf("failed to create kubernetes client: %w", err)) } @@ -91,13 +107,15 @@ func postWorkerInfo(s *state.State, r *http.Request) response.Response { } return response.SyncResponse(true, &apiv1.WorkerNodeInfoResponse{ - CA: clusterConfig.Certificates.CACert, + CA: cfg.Certificates.CACert, APIServers: servers, - ClusterCIDR: clusterConfig.Cluster.CIDR, + PodCIDR: cfg.Network.PodCIDR, KubeletToken: kubeletToken, KubeProxyToken: proxyToken, - ClusterDomain: clusterConfig.Kubelet.ClusterDomain, - ClusterDNS: clusterConfig.Kubelet.ClusterDNS, - CloudProvider: clusterConfig.Kubelet.CloudProvider, + ClusterDomain: cfg.Kubelet.ClusterDomain, + ClusterDNS: cfg.Kubelet.ClusterDNS, + CloudProvider: cfg.Kubelet.CloudProvider, + KubeletCert: workerCertificates.KubeletCert, + KubeletKey: workerCertificates.KubeletKey, }) } diff --git a/src/k8s/pkg/k8sd/app/app.go b/src/k8s/pkg/k8sd/app/app.go index bd1c908ff..09c1fa122 100644 --- a/src/k8s/pkg/k8sd/app/app.go +++ b/src/k8s/pkg/k8sd/app/app.go @@ -21,6 +21,8 @@ type Config struct { ListenPort uint // StateDir is the local directory to store the state of the node. StateDir string + // Snap is the snap instance to use. + Snap snap.Snap } // App is the k8sd microcluster instance. @@ -30,9 +32,12 @@ type App struct { // New initializes a new microcluster instance from configuration. func New(ctx context.Context, cfg Config) (*App, error) { - snapCtx := snap.ContextWithSnap(ctx, snap.NewDefaultSnap()) + ctx = snap.ContextWithSnap(ctx, cfg.Snap) - cluster, err := microcluster.App(snapCtx, microcluster.Args{ + if cfg.StateDir == "" { + cfg.StateDir = cfg.Snap.K8sdStateDir() + } + cluster, err := microcluster.App(ctx, microcluster.Args{ Verbose: cfg.Verbose, Debug: cfg.Debug, ListenPort: fmt.Sprintf("%d", cfg.ListenPort), diff --git a/src/k8s/pkg/k8sd/app/hooks.go b/src/k8s/pkg/k8sd/app/hooks.go deleted file mode 100644 index 3c868828d..000000000 --- a/src/k8s/pkg/k8sd/app/hooks.go +++ /dev/null @@ -1,97 +0,0 @@ -package app - -import ( - "fmt" - "log" - "path" - - "github.com/canonical/k8s/pkg/k8s/setup" - "github.com/canonical/k8s/pkg/snap" - "github.com/canonical/k8s/pkg/utils" - "github.com/canonical/k8s/pkg/utils/cert" - "github.com/canonical/microcluster/state" -) - -// onPostJoin is called when a control plane node joins the cluster. -// onPostJoin retrieves the cluster config from the database and configures local services. -func onPostJoin(s *state.State, initConfig map[string]string) error { - snap := snap.SnapFromContext(s.Context) - - clusterConfig, err := utils.GetClusterConfig(s.Context, s) - if err != nil { - return fmt.Errorf("failed to get cluster config: %w", err) - } - - if err := setup.InitFolders(snap.DataPath("args")); err != nil { - return fmt.Errorf("failed to setup folders: %w", err) - } - - if err := setup.InitServiceArgs(snap, map[string]map[string]string{ - "kube-apiserver": { - "--secure-port": fmt.Sprintf("%d", clusterConfig.APIServer.SecurePort), - }, - "kube-proxy": { - "--cluster-cidr": clusterConfig.Cluster.CIDR, - }, - }); err != nil { - return fmt.Errorf("failed to setup service arguments: %w", err) - } - - if err := setup.InitContainerd(snap); err != nil { - return fmt.Errorf("failed to initialize containerd: %w", err) - } - - caKeyPair, err := cert.NewCertKeyPairFromPEM([]byte(clusterConfig.Certificates.CACert), []byte(clusterConfig.Certificates.CAKey)) - if err != nil { - return fmt.Errorf("failed to create CA from pem: %w", err) - } - - if err := caKeyPair.SaveCertificate(path.Join(cert.KubePkiPath, "ca.crt")); err != nil { - return fmt.Errorf("failed to write CA cert: %w", err) - } - if err := caKeyPair.SavePrivateKey(path.Join(cert.KubePkiPath, "ca.key")); err != nil { - return fmt.Errorf("failed to write CA key: %w", err) - } - - certMan, err := setup.InitCertificates(caKeyPair) - if err != nil { - return fmt.Errorf("failed to setup certificates: %w", err) - } - - if err := setup.InitKubeconfigs(s.Context, s, certMan.CA, nil, &clusterConfig.APIServer.SecurePort); err != nil { - return fmt.Errorf("failed to generate kubeconfig files: %w", err) - } - - if err := setup.InitKubeApiserver(snap.Path("k8s/config/apiserver-token-hook.tmpl")); err != nil { - return fmt.Errorf("failed to initialize kube-apiserver: %w", err) - } - - if err := setup.InitPermissions(s.Context, snap); err != nil { - return fmt.Errorf("failed to setup permissions: %w", err) - } - - if err := setup.JoinK8sDqliteCluster(s.Context, s, snap); err != nil { - return fmt.Errorf("failed to join k8s-dqlite nodes: %w", err) - } - - if err := snap.StartService(s.Context, "k8s"); err != nil { - return fmt.Errorf("failed to start services: %w", err) - } - return nil -} - -func onPreRemove(s *state.State, force bool) error { - snap := snap.SnapFromContext(s.Context) - - // Remove k8s dqlite node from cluster. - // Fails if the k8s-dqlite cluster would not have a leader afterwards. - log.Println("Leave k8s-dqlite cluster") - err := setup.LeaveK8sDqliteCluster(s.Context, snap, s) - if err != nil { - return fmt.Errorf("failed to leave k8s-dqlite cluster: %w", err) - } - - // TODO: Remove node from kubernetes - - return nil -} diff --git a/src/k8s/pkg/k8sd/app/hooks_bootstrap.go b/src/k8s/pkg/k8sd/app/hooks_bootstrap.go index e204ea0da..cf0cfd650 100644 --- a/src/k8s/pkg/k8sd/app/hooks_bootstrap.go +++ b/src/k8s/pkg/k8sd/app/hooks_bootstrap.go @@ -7,16 +7,19 @@ import ( "database/sql" "encoding/json" "fmt" + "net" "net/http" "path" "time" apiv1 "github.com/canonical/k8s/api/v1" - "github.com/canonical/k8s/pkg/k8s/setup" + "github.com/canonical/k8s/pkg/k8sd/api/impl" "github.com/canonical/k8s/pkg/k8sd/database" + "github.com/canonical/k8s/pkg/k8sd/pki" + "github.com/canonical/k8s/pkg/k8sd/setup" "github.com/canonical/k8s/pkg/k8sd/types" "github.com/canonical/k8s/pkg/snap" - "github.com/canonical/k8s/pkg/utils/cert" + snaputil "github.com/canonical/k8s/pkg/snap/util" "github.com/canonical/k8s/pkg/utils/k8s" "github.com/canonical/microcluster/state" ) @@ -31,7 +34,7 @@ func onBootstrap(s *state.State, initConfig map[string]string) error { return onBootstrapControlPlane(s, initConfig) } -func onBootstrapWorkerNode(state *state.State, encodedToken string) error { +func onBootstrapWorkerNode(s *state.State, encodedToken string) error { token := &types.InternalWorkerNodeToken{} if err := token.Decode(encodedToken); err != nil { return fmt.Errorf("failed to parse worker token: %w", err) @@ -40,6 +43,10 @@ func onBootstrapWorkerNode(state *state.State, encodedToken string) error { if len(token.JoinAddresses) == 0 { return fmt.Errorf("empty list of control plane addresses") } + nodeIP := net.ParseIP(s.Address().Hostname()) + if nodeIP == nil { + return fmt.Errorf("failed to parse node IP address %s", s.Address().Hostname()) + } // TODO(neoaggelos): figure out how to use the microcluster client instead @@ -58,7 +65,7 @@ func onBootstrapWorkerNode(state *state.State, encodedToken string) error { Metadata apiv1.WorkerNodeInfoResponse `json:"metadata"` } - requestBody, err := json.Marshal(apiv1.WorkerNodeInfoRequest{Hostname: state.Name()}) + requestBody, err := json.Marshal(apiv1.WorkerNodeInfoRequest{Hostname: s.Name(), Address: nodeIP.String()}) if err != nil { return fmt.Errorf("failed to prepare worker info request: %w", err) } @@ -83,52 +90,51 @@ func onBootstrapWorkerNode(state *state.State, encodedToken string) error { } response := wrappedResp.Metadata - s := snap.SnapFromContext(state.Context) - if err := setup.InitFolders(s.DataPath("args")); err != nil { - return fmt.Errorf("failed to setup folders: %w", err) + snap := snap.SnapFromContext(s.Context) + + // Create directories + if err := setup.EnsureAllDirectories(snap); err != nil { + return fmt.Errorf("failed to create directories: %w", err) } - if err := setup.InitContainerd(s); err != nil { - return fmt.Errorf("failed to configure containerd: %w", err) + + // Certificates + certificates := &pki.WorkerNodePKI{ + CACert: response.CA, + KubeletCert: response.KubeletCert, + KubeletKey: response.KubeletKey, } - if err := setup.InitContainerdArgs(s, nil, nil); err != nil { - return fmt.Errorf("failed to configure containerd arguments: %w", err) + if err := certificates.CompleteCertificates(); err != nil { + return fmt.Errorf("failed to initialize cluster certificates: %w", err) } - if err := setup.WriteCA(s, response.CA); err != nil { - return fmt.Errorf("failed to write CA certificate: %w", err) + if err := setup.EnsureWorkerPKI(snap, certificates); err != nil { + return fmt.Errorf("failed to write cluster certificates: %w", err) } - kubeletArgs := map[string]string{ - "--hostname-override": state.Name(), - "--cluster-dns": response.ClusterDNS, - "--cluster-domain": response.ClusterDomain, - "--cloud-provider": response.CloudProvider, + // Kubeconfigs + if err := setup.Kubeconfig(path.Join(snap.KubernetesConfigDir(), "kubelet.conf"), response.KubeletToken, "127.0.0.1:6443", certificates.CACert); err != nil { + return fmt.Errorf("failed to generate kubelet kubeconfig: %w", err) } - if err := setup.InitKubeletArgs(s, kubeletArgs, nil); err != nil { - return fmt.Errorf("failed to configure kubelet: %w", err) - } - if err := setup.RenderKubeletKubeconfig(s, response.KubeletToken, response.CA); err != nil { - return fmt.Errorf("failed to render kubelet kubeconfig: %w", err) + if err := setup.Kubeconfig(path.Join(snap.KubernetesConfigDir(), "proxy.conf"), response.KubeProxyToken, "127.0.0.1:6443", certificates.CACert); err != nil { + return fmt.Errorf("failed to generate kube-proxy kubeconfig: %w", err) } - proxyArgs := map[string]string{ - "--hostname-override": state.Name(), - "--cluster-cidr": response.ClusterCIDR, + // Worker node services + if err := setup.Containerd(snap); err != nil { + return fmt.Errorf("failed to configure containerd: %w", err) } - if err := setup.InitKubeProxyArgs(s, proxyArgs, nil); err != nil { - return fmt.Errorf("failed to configure kube-proxy: %w", err) + if err := setup.Kubelet(snap, s.Name(), nodeIP, response.ClusterDNS, response.ClusterDomain, response.CloudProvider); err != nil { + return fmt.Errorf("failed to configure kubelet: %w", err) } - if err := setup.RenderKubeProxyKubeconfig(s, response.KubeProxyToken, response.CA); err != nil { - return fmt.Errorf("failed to render kube-proxy kubeconfig: %w", err) + if err := setup.KubeProxy(snap, s.Name(), response.PodCIDR); err != nil { + return fmt.Errorf("failed to configure kube-proxy: %w", err) } - - if err := setup.InitAPIServerProxy(s, response.APIServers); err != nil { - return fmt.Errorf("failed to configure k8s-apiserver-proxy: %w", err) + if err := setup.K8sAPIServerProxy(snap, response.APIServers); err != nil { + return fmt.Errorf("failed to configure kube-proxy: %w", err) } - // TODO: mark node as worker - - if err := snap.StartWorkerServices(state.Context, s); err != nil { - return fmt.Errorf("failed to start services: %w", err) + // Start services + if err := snaputil.StartWorkerServices(s.Context, snap); err != nil { + return fmt.Errorf("failed to start worker services: %w", err) } return nil @@ -137,93 +143,125 @@ func onBootstrapWorkerNode(state *state.State, encodedToken string) error { func onBootstrapControlPlane(s *state.State, initConfig map[string]string) error { snap := snap.SnapFromContext(s.Context) - err := setup.InitFolders(snap.DataPath("args")) + bootstrapConfig, err := apiv1.BootstrapConfigFromMap(initConfig) if err != nil { - return fmt.Errorf("failed to setup folders: %w", err) + return fmt.Errorf("failed to unmarshal bootstrap config: %w", err) } - - err = setup.InitServiceArgs(snap, nil) + cfg, err := types.MergeClusterConfig(types.DefaultClusterConfig(), types.ClusterConfigFromBootstrapConfig(bootstrapConfig)) if err != nil { - return fmt.Errorf("failed to setup service arguments: %w", err) + return fmt.Errorf("failed initialize cluster config from bootstrap config: %w", err) } - - if err := setup.InitContainerd(snap); err != nil { - return fmt.Errorf("failed to initialize containerd: %w", err) - } - - certMan, err := setup.InitCertificates(nil) - if err != nil { - return fmt.Errorf("failed to setup certificates: %w", err) + nodeIP := net.ParseIP(s.Address().Hostname()) + if nodeIP == nil { + return fmt.Errorf("failed to parse node IP address %q", s.Address().Hostname()) } + certificates := pki.NewControlPlanePKI(pki.ControlPlanePKIOpts{ + Hostname: s.Name(), + IPSANs: []net.IP{nodeIP}, + Years: 10, + AllowSelfSignedCA: true, + }) - err = setup.InitKubeconfigs(s.Context, s, certMan.CA, nil, nil) - if err != nil { - return fmt.Errorf("failed to kubeconfig files: %w", err) + // Create directories + if err := setup.EnsureAllDirectories(snap); err != nil { + return fmt.Errorf("failed to create directories: %w", err) + } + + // Certificates + if err := certificates.CompleteCertificates(); err != nil { + return fmt.Errorf("failed to initialize cluster certificates: %w", err) + } + if err := setup.EnsureControlPlanePKI(snap, certificates); err != nil { + return fmt.Errorf("failed to write cluster certificates: %w", err) + } + + // Add certificates to the cluster config + cfg.Certificates.CACert = certificates.CACert + cfg.Certificates.CAKey = certificates.CAKey + cfg.Certificates.FrontProxyCACert = certificates.FrontProxyCACert + cfg.Certificates.FrontProxyCAKey = certificates.FrontProxyCAKey + cfg.Certificates.APIServerKubeletClientCert = certificates.APIServerKubeletClientCert + cfg.Certificates.APIServerKubeletClientKey = certificates.APIServerKubeletClientKey + cfg.Certificates.K8sDqliteCert = certificates.K8sDqliteCert + cfg.Certificates.K8sDqliteKey = certificates.K8sDqliteKey + cfg.APIServer.ServiceAccountKey = certificates.ServiceAccountKey + + // Generate kubeconfigs + for _, kubeconfig := range []struct { + file string + username string + groups []string + }{ + {file: "admin.conf", username: "kubernetes-admin", groups: []string{"system:masters"}}, + {file: "controller.conf", username: "system:kube-controller-manager"}, + {file: "proxy.conf", username: "system:kube-proxy"}, + {file: "scheduler.conf", username: "system:kube-scheduler"}, + {file: "kubelet.conf", username: fmt.Sprintf("system:node:%s", s.Name()), groups: []string{"system:nodes"}}, + } { + token, err := impl.GetOrCreateAuthToken(s.Context, s, kubeconfig.username, kubeconfig.groups) + if err != nil { + return fmt.Errorf("failed to generate token for username=%s groups=%v: %w", kubeconfig.username, kubeconfig.groups, err) + } + if err := setup.Kubeconfig(path.Join(snap.KubernetesConfigDir(), kubeconfig.file), token, fmt.Sprintf("127.0.0.1:%d", cfg.APIServer.SecurePort), cfg.Certificates.CACert); err != nil { + return fmt.Errorf("failed to write kubeconfig %s: %w", kubeconfig.file, err) + } + } + + // Configure datastore + switch cfg.APIServer.Datastore { + case "k8s-dqlite": + if err := setup.K8sDqlite(snap, fmt.Sprintf("%s:%d", nodeIP.String(), cfg.K8sDqlite.Port), nil); err != nil { + return fmt.Errorf("failed to configure k8s-dqlite: %w", err) + } + default: + return fmt.Errorf("unknown datastore %q, must be k8s-dqlite", cfg.APIServer.Datastore) + } + + // Configure services + if err := setup.Containerd(snap); err != nil { + return fmt.Errorf("failed to configure containerd: %w", err) } - - err = setup.InitKubeApiserver(snap.Path("k8s/config/apiserver-token-hook.tmpl")) - if err != nil { - return fmt.Errorf("failed to initialize kube-apiserver: %w", err) + if err := setup.Kubelet(snap, s.Name(), nodeIP, cfg.Kubelet.ClusterDNS, cfg.Kubelet.ClusterDomain, cfg.Kubelet.CloudProvider); err != nil { + return fmt.Errorf("failed to configure kubelet: %w", err) } - - err = setup.InitPermissions(s.Context, snap) - if err != nil { - return fmt.Errorf("failed to setup permissions: %w", err) + if err := setup.KubeProxy(snap, s.Name(), cfg.Network.PodCIDR); err != nil { + return fmt.Errorf("failed to configure kube-proxy: %w", err) } - - clusterConfig := types.DefaultClusterConfig() - bootstrapConfig, err := apiv1.BootstrapConfigFromMap(initConfig) - if err != nil { - return fmt.Errorf("failed to unmarshal bootstrap config: %w", err) + if err := setup.KubeControllerManager(snap); err != nil { + return fmt.Errorf("failed to configure kube-controller-manager: %w", err) } - - // Set k8s-dqlite configuration - k8sDqliteCertPair, err := cert.LoadCertKeyPair(snap.CommonPath(cert.K8sDqlitePkiPath, "cluster.key"), snap.CommonPath(cert.K8sDqlitePkiPath, "cluster.crt")) - if err != nil { - return fmt.Errorf("failed to load k8s-dqlite cert-key pair: %w", err) + if err := setup.KubeScheduler(snap); err != nil { + return fmt.Errorf("failed to configure kube-scheduler: %w", err) } - clusterConfig.Certificates.K8sDqliteCert = string(k8sDqliteCertPair.CertPem) - clusterConfig.Certificates.K8sDqliteKey = string(k8sDqliteCertPair.KeyPem) - - caPair, err := cert.LoadCertKeyPair(path.Join(cert.KubePkiPath, "ca.key"), path.Join(cert.KubePkiPath, "ca.crt")) - if err != nil { - return fmt.Errorf("failed to load k8s-dqlite cert-key pair: %w", err) + if err := setup.KubeAPIServer(snap, cfg.Network.ServiceCIDR, s.Address().Path("1.0", "kubernetes", "auth", "webhook").String(), true, cfg.APIServer.Datastore, cfg.APIServer.AuthorizationMode); err != nil { + return fmt.Errorf("failed to configure kube-apiserver: %w", err) } - clusterConfig.Certificates.CACert = string(caPair.CertPem) - clusterConfig.Certificates.CAKey = string(caPair.KeyPem) - clusterConfig, err = types.MergeClusterConfig(clusterConfig, types.ClusterConfigFromBootstrapConfig(bootstrapConfig)) - if err != nil { - return fmt.Errorf("failed to merge cluster config with bootstrap config: %w", err) + // Write cluster configuration to dqlite + if err := s.Database.Transaction(s.Context, func(ctx context.Context, tx *sql.Tx) error { + if err := database.SetClusterConfig(ctx, tx, cfg); err != nil { + return fmt.Errorf("failed to write cluster configuration: %w", err) + } + return nil + }); err != nil { + return fmt.Errorf("database transaction to update cluster configuration failed: %w", err) } - // TODO(neoaggelos): first generate config then reconcile state - s.Database.Transaction(s.Context, func(ctx context.Context, tx *sql.Tx) error { - return database.SetClusterConfig(ctx, tx, clusterConfig) - }) - - k8sDqliteInit := setup.K8sDqliteInit{ - Address: fmt.Sprintf("%s:%d", s.Address().Hostname(), clusterConfig.K8sDqlite.Port), - } - if err := setup.WriteClusterInitFile(k8sDqliteInit); err != nil { - return fmt.Errorf("failed to write cluster init file: %w", err) + // Start services + if err := snaputil.StartControlPlaneServices(s.Context, snap); err != nil { + return fmt.Errorf("failed to start control plane services: %w", err) } - err = snap.StartService(s.Context, "k8s") - if err != nil { - return fmt.Errorf("failed to start services: %w", err) - } - k8sClient, err := k8s.NewClient() + // Wait for API server to come up + k8sClient, err := k8s.NewClient(snap) if err != nil { return fmt.Errorf("failed to create k8s client: %w", err) } - // The apiserver needs to be ready to start components. err = k8s.WaitApiServerReady(s.Context, k8sClient) if err != nil { return fmt.Errorf("k8s api server did not become ready in time: %w", err) } - // TODO: start configured components. return nil } diff --git a/src/k8s/pkg/k8sd/app/hooks_join.go b/src/k8s/pkg/k8sd/app/hooks_join.go new file mode 100644 index 000000000..12479d0c5 --- /dev/null +++ b/src/k8s/pkg/k8sd/app/hooks_join.go @@ -0,0 +1,162 @@ +package app + +import ( + "fmt" + "log" + "net" + "path" + + old_setup "github.com/canonical/k8s/pkg/k8s/setup" + "github.com/canonical/k8s/pkg/k8sd/api/impl" + "github.com/canonical/k8s/pkg/k8sd/pki" + "github.com/canonical/k8s/pkg/k8sd/setup" + "github.com/canonical/k8s/pkg/snap" + snaputil "github.com/canonical/k8s/pkg/snap/util" + "github.com/canonical/k8s/pkg/utils" + "github.com/canonical/k8s/pkg/utils/k8s" + "github.com/canonical/microcluster/state" +) + +// onPostJoin is called when a control plane node joins the cluster. +// onPostJoin retrieves the cluster config from the database and configures local services. +func onPostJoin(s *state.State, initConfig map[string]string) error { + snap := snap.SnapFromContext(s.Context) + + cfg, err := utils.GetClusterConfig(s.Context, s) + if err != nil { + return fmt.Errorf("failed to get cluster config: %w", err) + } + nodeIP := net.ParseIP(s.Address().Hostname()) + if nodeIP == nil { + return fmt.Errorf("failed to parse node IP address %q", s.Address().Hostname()) + } + + // Create directories + if err := setup.EnsureAllDirectories(snap); err != nil { + return fmt.Errorf("failed to create directories: %w", err) + } + + // Certificates + certificates := pki.NewControlPlanePKI(pki.ControlPlanePKIOpts{ + Hostname: s.Name(), + IPSANs: []net.IP{nodeIP}, + Years: 10, + }) + + // load existing certificates, then generate certificates for the node + certificates.CACert = cfg.Certificates.CACert + certificates.CAKey = cfg.Certificates.CAKey + certificates.FrontProxyCACert = cfg.Certificates.FrontProxyCACert + certificates.FrontProxyCAKey = cfg.Certificates.FrontProxyCAKey + certificates.APIServerKubeletClientCert = cfg.Certificates.APIServerKubeletClientCert + certificates.APIServerKubeletClientKey = cfg.Certificates.APIServerKubeletClientKey + certificates.K8sDqliteCert = cfg.Certificates.K8sDqliteCert + certificates.K8sDqliteKey = cfg.Certificates.K8sDqliteKey + certificates.ServiceAccountKey = cfg.APIServer.ServiceAccountKey + + if err := certificates.CompleteCertificates(); err != nil { + return fmt.Errorf("failed to initialize cluster certificates: %w", err) + } + if err := setup.EnsureControlPlanePKI(snap, certificates); err != nil { + return fmt.Errorf("failed to write cluster certificates: %w", err) + } + + // Generate kubeconfigs + for _, kubeconfig := range []struct { + file string + username string + groups []string + }{ + {file: "admin.conf", username: "kubernetes-admin", groups: []string{"system:masters"}}, + {file: "controller.conf", username: "system:kube-controller-manager"}, + {file: "proxy.conf", username: "system:kube-proxy"}, + {file: "scheduler.conf", username: "system:kube-scheduler"}, + {file: "kubelet.conf", username: fmt.Sprintf("system:node:%s", s.Name()), groups: []string{"system:nodes"}}, + } { + token, err := impl.GetOrCreateAuthToken(s.Context, s, kubeconfig.username, kubeconfig.groups) + if err != nil { + return fmt.Errorf("failed to generate token for username=%s groups=%v: %w", kubeconfig.username, kubeconfig.groups, err) + } + if err := setup.Kubeconfig(path.Join(snap.KubernetesConfigDir(), kubeconfig.file), token, fmt.Sprintf("127.0.0.1:%d", cfg.APIServer.SecurePort), cfg.Certificates.CACert); err != nil { + return fmt.Errorf("failed to write kubeconfig %s: %w", kubeconfig.file, err) + } + } + + // Configure datastore + switch cfg.APIServer.Datastore { + case "k8s-dqlite": + leader, err := s.Leader() + if err != nil { + return fmt.Errorf("failed to get dqlite leader: %w", err) + } + members, err := leader.GetClusterMembers(s.Context) + if err != nil { + return fmt.Errorf("failed to get microcluster members: %w", err) + } + cluster := make([]string, len(members)) + for _, member := range members { + cluster = append(cluster, fmt.Sprintf("%s:%d", member.Address.Addr(), cfg.K8sDqlite.Port)) + } + + address := fmt.Sprintf("%s:%d", nodeIP.String(), cfg.K8sDqlite.Port) + if err := setup.K8sDqlite(snap, address, cluster); err != nil { + return fmt.Errorf("failed to configure k8s-dqlite with address=%s cluster=%v: %w", address, cluster, err) + } + default: + return fmt.Errorf("unknown datastore %q, must be k8s-dqlite", cfg.APIServer.Datastore) + } + + // Configure services + if err := setup.Containerd(snap); err != nil { + return fmt.Errorf("failed to configure containerd: %w", err) + } + if err := setup.Kubelet(snap, s.Name(), nodeIP, cfg.Kubelet.ClusterDNS, cfg.Kubelet.ClusterDomain, cfg.Kubelet.CloudProvider); err != nil { + return fmt.Errorf("failed to configure kubelet: %w", err) + } + if err := setup.KubeProxy(snap, s.Name(), cfg.Network.PodCIDR); err != nil { + return fmt.Errorf("failed to configure kube-proxy: %w", err) + } + if err := setup.KubeControllerManager(snap); err != nil { + return fmt.Errorf("failed to configure kube-controller-manager: %w", err) + } + if err := setup.KubeScheduler(snap); err != nil { + return fmt.Errorf("failed to configure kube-scheduler: %w", err) + } + if err := setup.KubeAPIServer(snap, cfg.Network.ServiceCIDR, s.Address().Path("1.0", "kubernetes", "auth", "webhook").String(), true, cfg.APIServer.Datastore, cfg.APIServer.AuthorizationMode); err != nil { + return fmt.Errorf("failed to configure kube-apiserver: %w", err) + } + + // Start services + if err := snaputil.StartControlPlaneServices(s.Context, snap); err != nil { + return fmt.Errorf("failed to start control plane services: %w", err) + } + + // Wait for API server to come up + k8sClient, err := k8s.NewClient(snap) + if err != nil { + return fmt.Errorf("failed to create k8s client: %w", err) + } + + err = k8s.WaitApiServerReady(s.Context, k8sClient) + if err != nil { + return fmt.Errorf("k8s api server did not become ready in time: %w", err) + } + + return nil +} + +func onPreRemove(s *state.State, force bool) error { + snap := snap.SnapFromContext(s.Context) + + // Remove k8s dqlite node from cluster. + // Fails if the k8s-dqlite cluster would not have a leader afterwards. + log.Println("Leave k8s-dqlite cluster") + err := old_setup.LeaveK8sDqliteCluster(s.Context, snap, s) + if err != nil { + return fmt.Errorf("failed to leave k8s-dqlite cluster: %w", err) + } + + // TODO: Remove node from kubernetes + + return nil +} diff --git a/src/k8s/pkg/k8sd/pki/control_plane.go b/src/k8s/pkg/k8sd/pki/control_plane.go new file mode 100644 index 000000000..a42e11fcf --- /dev/null +++ b/src/k8s/pkg/k8sd/pki/control_plane.go @@ -0,0 +1,223 @@ +package pki + +import ( + "crypto/x509/pkix" + "fmt" + "net" +) + +// ControlPlanePKI is a list of all certificates we require for a control plane node. +type ControlPlanePKI struct { + allowSelfSignedCA bool // create self-signed CA certificates if missing + hostname string // node name + ipSANs []net.IP // IP SANs for generated certificates + dnsSANs []string // DNS SANs for the certificates below + years int // how many years the generated certificates will be valid for + + CACert, CAKey string // CN=kubernetes-ca (self-signed) + FrontProxyCACert, FrontProxyCAKey string // CN=kubernetes-front-proxy-ca (self-signed) + FrontProxyClientCert, FrontProxyClientKey string // CN=front-proxy-client (signed by kubernetes-front-proxy-ca) + ServiceAccountKey string // private key used to sign service account tokens + + // CN=k8s-dqlite, DNS=hostname, IP=127.0.0.1 (self-signed) + K8sDqliteCert, K8sDqliteKey string + + // CN=kube-apiserver, DNS=hostname,kubernetes.* IP=127.0.0.1,10.152.183.1,address (signed by kubernetes-ca) + APIServerCert, APIServerKey string + + // CN=kube-apiserver-kubelet-client, O=system:masters (signed by kubernetes-ca) + APIServerKubeletClientCert, APIServerKubeletClientKey string + + // CN=system:node:hostname, O=system:nodes, DNS=hostname, IP=127.0.0.1,address (signed by kubernetes-ca) + KubeletCert, KubeletKey string +} + +type ControlPlanePKIOpts struct { + Hostname string + DNSSANs []string + IPSANs []net.IP + Years int + AllowSelfSignedCA bool +} + +func NewControlPlanePKI(opts ControlPlanePKIOpts) *ControlPlanePKI { + if opts.Years == 0 { + opts.Years = 1 + } + + return &ControlPlanePKI{ + allowSelfSignedCA: opts.AllowSelfSignedCA, + hostname: opts.Hostname, + years: opts.Years, + ipSANs: opts.IPSANs, + dnsSANs: opts.DNSSANs, + } +} + +// CompleteCertificates generates missing or unset certificates. If only a certificate is set and not a key, we assume that the cluster is using managed certificates. +func (c *ControlPlanePKI) CompleteCertificates() error { + // Fail hard if keys of self-signed certificates are set without the respective certificates + switch { + case c.CACert == "" && c.CAKey != "": + return fmt.Errorf("kubernetes CA key is set without a certificate, fail to prevent causing issues") + case c.FrontProxyCACert == "" && c.FrontProxyCAKey != "": + return fmt.Errorf("front-proxy CA key is set without a certificate, fail to prevent causing issues") + case c.K8sDqliteCert == "" && c.K8sDqliteKey != "": + return fmt.Errorf("k8s-dqlite certificate key set without a certificate, fail to prevent further issues") + case c.K8sDqliteCert != "" && c.K8sDqliteKey == "": + return fmt.Errorf("k8s-dqlite certificate set without a key, fail to prevent further issues") + } + + // Generate self-signed CA (if not set already) + if c.CACert == "" && c.CAKey == "" { + if !c.allowSelfSignedCA { + return fmt.Errorf("kubernetes CA not specified and generating self-signed CA not allowed") + } + cert, key, err := generateSelfSignedCA(pkix.Name{CommonName: "kubernetes-ca"}, c.years, 2048) + if err != nil { + return fmt.Errorf("failed to generate kubernetes CA: %w", err) + } + c.CACert = cert + c.CAKey = key + } + + caCertificate, caPrivateKey, err := loadCertificate(c.CACert, c.CAKey) + if err != nil { + return fmt.Errorf("failed to parse kubernetes CA: %w", err) + } + + // Generate self-signed CA for front-proxy (if not set already) + if c.FrontProxyCACert == "" && c.FrontProxyCAKey == "" { + if !c.allowSelfSignedCA { + return fmt.Errorf("front-proxy CA not specified and generating self-signed CA not allowed") + } + cert, key, err := generateSelfSignedCA(pkix.Name{CommonName: "front-proxy-ca"}, c.years, 2048) + if err != nil { + return fmt.Errorf("failed to generate front-proxy CA: %w", err) + } + c.FrontProxyCACert = cert + c.FrontProxyCAKey = key + } + + // Generate front proxy client certificate (ok to override) + if c.FrontProxyClientCert == "" || c.FrontProxyClientKey == "" { + frontProxyCACert, frontProxyCAKey, err := loadCertificate(c.FrontProxyCACert, c.FrontProxyCAKey) + switch { + case err != nil: + return fmt.Errorf("failed to parse front proxy CA: %w", err) + case frontProxyCAKey == nil: + return fmt.Errorf("using an external front proxy CA without providing the front-proxy-client certificate is not possible") + } + + template, err := generateCertificate(pkix.Name{CommonName: "front-proxy-client"}, c.years, false, nil, nil) + if err != nil { + return fmt.Errorf("failed to generate front-proxy-client certificate: %w", err) + } + cert, key, err := signCertificate(template, 2048, frontProxyCACert, &frontProxyCAKey.PublicKey, frontProxyCAKey) + if err != nil { + return fmt.Errorf("failed to sign front-proxy-client certificate: %w", err) + } + + c.FrontProxyClientCert = cert + c.FrontProxyClientKey = key + } + + // Generate k8s-dqlite client certificate (if missing) + if c.K8sDqliteCert == "" && c.K8sDqliteKey == "" { + if !c.allowSelfSignedCA { + return fmt.Errorf("k8s-dqlite certificate not specified and generating self-signed certificates is not allowed") + } + + template, err := generateCertificate(pkix.Name{CommonName: "k8s"}, c.years, false, append(c.dnsSANs, c.hostname), append(c.ipSANs, net.IP{127, 0, 0, 1})) + if err != nil { + return fmt.Errorf("failed to generate k8s-dqlite certificate: %w", err) + } + cert, key, err := signCertificate(template, 2048, template, nil, nil) + if err != nil { + return fmt.Errorf("failed to self-sign k8s-dqlite certificate: %w", err) + } + + c.K8sDqliteCert = cert + c.K8sDqliteKey = key + } + + // Generate service account key (if missing) + if c.ServiceAccountKey == "" { + if !c.allowSelfSignedCA { + return fmt.Errorf("service account signing key not specified and generating new key is not allowed") + } + + key, err := generateKey(2048) + if err != nil { + return fmt.Errorf("failed to generate service account key: %w", err) + } + + c.ServiceAccountKey = key + } + + // Generate kubelet certificate (if missing) + if c.KubeletCert == "" || c.KubeletKey == "" { + if caPrivateKey == nil { + return fmt.Errorf("using an external kubernetes CA without providing the kubelet certificate is not possible") + } + + template, err := generateCertificate( + pkix.Name{CommonName: fmt.Sprintf("system:node:%s", c.hostname), Organization: []string{"system:nodes"}}, + c.years, false, append(c.dnsSANs, c.hostname), append(c.ipSANs, net.IP{127, 0, 0, 1}), + ) + if err != nil { + return fmt.Errorf("failed to generate kubelet certificate: %w", err) + } + cert, key, err := signCertificate(template, 2048, caCertificate, &caPrivateKey.PublicKey, caPrivateKey) + if err != nil { + return fmt.Errorf("failed to sign kubelet certificate: %w", err) + } + + c.KubeletCert = cert + c.KubeletKey = key + } + + // Generate apiserver-kubelet-client certificate (if missing) + if c.APIServerKubeletClientCert == "" || c.APIServerKubeletClientKey == "" { + if caPrivateKey == nil { + return fmt.Errorf("using an external kubernetes CA without providing the apiserver-kubelet-client certificate is not possible") + } + + template, err := generateCertificate(pkix.Name{CommonName: "apiserver-kubelet-client", Organization: []string{"system:masters"}}, c.years, false, nil, nil) + if err != nil { + return fmt.Errorf("failed to generate apiserver-kubelet-client certificate: %w", err) + } + cert, key, err := signCertificate(template, 2048, caCertificate, &caPrivateKey.PublicKey, caPrivateKey) + if err != nil { + return fmt.Errorf("failed to sign apiserver-kubelet-client certificate: %w", err) + } + + c.APIServerKubeletClientCert = cert + c.APIServerKubeletClientKey = key + } + + // Generate kube-apiserver certificate (if missing) + if c.APIServerCert == "" || c.APIServerKey == "" { + if caPrivateKey == nil { + return fmt.Errorf("using an external kubernetes CA without providing the apiserver certificate is not possible") + } + + // TODO(neoaggelos): we also need to specify the kubernetes service IP here, not hardcode 10.152.183.1 + template, err := generateCertificate( + pkix.Name{CommonName: "kube-apiserver"}, + c.years, + false, + append(c.dnsSANs, "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", "kubernetes.default.svc.cluster.local"), append(c.ipSANs, net.IP{10, 152, 183, 1}, net.IP{127, 0, 0, 1})) + if err != nil { + return fmt.Errorf("failed to generate apiserver certificate: %w", err) + } + cert, key, err := signCertificate(template, 2048, caCertificate, &caPrivateKey.PublicKey, caPrivateKey) + if err != nil { + return fmt.Errorf("failed to sign apiserver certificate: %w", err) + } + + c.APIServerCert = cert + c.APIServerKey = key + } + return nil +} diff --git a/src/k8s/pkg/k8sd/pki/control_plane_test.go b/src/k8s/pkg/k8sd/pki/control_plane_test.go new file mode 100644 index 000000000..4b24a0564 --- /dev/null +++ b/src/k8s/pkg/k8sd/pki/control_plane_test.go @@ -0,0 +1,54 @@ +package pki_test + +import ( + "testing" + + "github.com/canonical/k8s/pkg/k8sd/pki" + . "github.com/onsi/gomega" +) + +func TestControlPlaneCertificates(t *testing.T) { + c := pki.NewControlPlanePKI(pki.ControlPlanePKIOpts{ + Hostname: "h1", + Years: 10, + AllowSelfSignedCA: true, + }) + + g := NewWithT(t) + + g.Expect(c.CompleteCertificates()).To(BeNil()) + g.Expect(c.CompleteCertificates()).To(BeNil()) + + t.Run("MissingCAKey", func(t *testing.T) { + c := pki.NewControlPlanePKI(pki.ControlPlanePKIOpts{ + Hostname: "h1", + Years: 10, + }) + c.CACert = ` +-----BEGIN CERTIFICATE----- +MIIDtTCCAp2gAwIBAgIQOPOTOjxvIVlC5ev8EzrnITANBgkqhkiG9w0BAQsFADAY +MRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMB4XDTI0MDIwODAyNDYyOVoXDTM0MDIw +ODAyNDYyOVowGTEXMBUGA1UEAxMOa3ViZS1hcGlzZXJ2ZXIwggEiMA0GCSqGSIb3 +DQEBAQUAA4IBDwAwggEKAoIBAQCum3KkohfK+E4KCpauilnlxm0e6y+jzyOaRCHx +P/3iLqN5zN+s2SV+GJNNcT3vSVZ1YhcJKWNrs7QxK2qcq9OhHncmp9Vqu5BV9O+e +ys4bBlf08lHH0//wrAwXy71ueWXN2uWyFg4i2VSirbRxpXGIR751i4qVtutbSOPy +3Jjf07upq3zAMyvTx1YTZcwduwW2vrU1f48IZOTueS1eOz0YjCkWLueD2uhLLgRA +mcxq33pwTM9P0MaZGrrM2GeA+1Hyss5WtoEMkR6TPUWQmYcKFEZui9/JpLfbM8yu +6h6Ta7GeSccjtclHSGp9fge0IXErhYSmLNoQ7JP8fQeg0DpTAgMBAAGjgfkwgfYw +DgYDVR0PAQH/BAQDAgSwMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATAM +BgNVHRMBAf8EAjAAMB8GA1UdIwQYMBaAFJjD6HMwGRJQMOzNm919/ZaqdcUwMIGV +BgNVHREEgY0wgYqCCmt1YmVybmV0ZXOCEmt1YmVybmV0ZXMuZGVmYXVsdIIWa3Vi +ZXJuZXRlcy5kZWZhdWx0LnN2Y4Iea3ViZXJuZXRlcy5kZWZhdWx0LnN2Yy5jbHVz +dGVygiRrdWJlcm5ldGVzLmRlZmF1bHQuc3ZjLmNsdXN0ZXIubG9jYWyHBAqYtwGH +BH8AAAEwDQYJKoZIhvcNAQELBQADggEBADPWn//rPb0SmZ49WhIa6wc39Ryl7eGo +Q2H+RY9BMah/xge6fLgeTvFe+H6Vol9BVqm5XgD0BuD5yzKYI2aDq8Ikm4EMOxPl +7Gs9cqWMMF7Iiw+rYJY4vwzm+5kSCg6oxBx8GLYYkDpbFe8UAWKf/9QTghtoBEEw +JVBDECnQwJU4tb9ANmPbgxmCYLZjx2vmXQRlXpe6QS9nPmMSS9KkJMyLEEpgzIIA +aSprnA8WIeSaO/5wLMYS1lUWWzegz2LnKuJ5C5Q+XYkwIY/vFH7OSTnmvt+rHwhh +4Oj+ScJ0RKnGGcXQnctSvMogDoucw7Y2RjxKcJV8fEKV5ZIeTz0U+nE= +-----END CERTIFICATE-----` + + g := NewWithT(t) + g.Expect(c.CompleteCertificates()).ToNot(BeNil()) + }) +} diff --git a/src/k8s/pkg/k8sd/pki/generate.go b/src/k8s/pkg/k8sd/pki/generate.go new file mode 100644 index 000000000..2da317680 --- /dev/null +++ b/src/k8s/pkg/k8sd/pki/generate.go @@ -0,0 +1,115 @@ +package pki + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "net" + "time" +) + +// generateSerialNumber returns a random number that can be used for the SerialNumber field in an x509 certificate. +func generateSerialNumber() (*big.Int, error) { + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + return nil, err + } + return serialNumber, nil +} + +func generateCertificate(subject pkix.Name, years int, ca bool, dnsSANs []string, ipSANs []net.IP) (*x509.Certificate, error) { + serialNumber, err := generateSerialNumber() + if err != nil { + return nil, fmt.Errorf("failed to generate serial number for certificate template: %w", err) + } + + cert := &x509.Certificate{ + SerialNumber: serialNumber, + Subject: subject, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(years, 0, 0), + IPAddresses: ipSANs, + DNSNames: dnsSANs, + BasicConstraintsValid: true, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + } + if ca { + cert.IsCA = true + cert.KeyUsage = x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign + } else { + cert.IsCA = false + cert.KeyUsage = x509.KeyUsageKeyEncipherment | x509.KeyUsageDataEncipherment | x509.KeyUsageDigitalSignature + } + + return cert, nil +} + +func generateSelfSignedCA(subject pkix.Name, years int, bits int) (string, string, error) { + cert, err := generateCertificate(subject, years, true, nil, nil) + if err != nil { + return "", "", fmt.Errorf("failed to generate certificate: %w", err) + } + + key, err := rsa.GenerateKey(rand.Reader, bits) + if err != nil { + return "", "", fmt.Errorf("failed to generate RSA private key: %w", err) + } + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}) + if keyPEM == nil { + return "", "", fmt.Errorf("failed to encode private key PEM") + } + derBytes, err := x509.CreateCertificate(rand.Reader, cert, cert, &key.PublicKey, key) + if err != nil { + return "", "", fmt.Errorf("failed to self-sign certificate: %w", err) + } + crtPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) + if crtPEM == nil { + return "", "", fmt.Errorf("failed to encode certificate PEM") + } + + return string(crtPEM), string(keyPEM), nil +} + +func signCertificate(certificate *x509.Certificate, bits int, parent *x509.Certificate, pub any, priv any) (string, string, error) { + key, err := rsa.GenerateKey(rand.Reader, bits) + if err != nil { + return "", "", fmt.Errorf("failed to generate RSA private key: %w", err) + } + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}) + if keyPEM == nil { + return "", "", fmt.Errorf("failed to encode private key PEM") + } + + if pub == nil && priv == nil { + priv = key + pub = &key.PublicKey + } + + derBytes, err := x509.CreateCertificate(rand.Reader, certificate, parent, &key.PublicKey, priv) + if err != nil { + return "", "", fmt.Errorf("failed to sign certificate: %w", err) + } + crtPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) + if crtPEM == nil { + return "", "", fmt.Errorf("failed to encode certificate PEM") + } + + return string(crtPEM), string(keyPEM), nil +} + +func generateKey(bits int) (string, error) { + key, err := rsa.GenerateKey(rand.Reader, bits) + if err != nil { + return "", fmt.Errorf("failed to generate RSA private key: %w", err) + } + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}) + if keyPEM == nil { + return "", fmt.Errorf("failed to encode private key PEM") + } + return string(keyPEM), nil +} diff --git a/src/k8s/pkg/k8sd/pki/generate_test.go b/src/k8s/pkg/k8sd/pki/generate_test.go new file mode 100644 index 000000000..0f8ea17c3 --- /dev/null +++ b/src/k8s/pkg/k8sd/pki/generate_test.go @@ -0,0 +1,31 @@ +package pki + +import ( + "crypto/x509/pkix" + "testing" + + . "github.com/onsi/gomega" +) + +func Test_generateSelfSignedCA(t *testing.T) { + cert, key, err := generateSelfSignedCA(pkix.Name{CommonName: "test-cert"}, 10, 2048) + + g := NewWithT(t) + g.Expect(err).To(BeNil()) + g.Expect(cert).ToNot(BeEmpty()) + g.Expect(key).ToNot(BeEmpty()) + + t.Run("Load", func(t *testing.T) { + c, k, err := loadCertificate(cert, key) + g.Expect(err).To(BeNil()) + g.Expect(c).ToNot(BeNil()) + g.Expect(k).ToNot(BeNil()) + }) + + t.Run("LoadCertOnly", func(t *testing.T) { + cert, key, err := loadCertificate(cert, "") + g.Expect(err).To(BeNil()) + g.Expect(cert).ToNot(BeNil()) + g.Expect(key).To(BeNil()) + }) +} diff --git a/src/k8s/pkg/k8sd/pki/load.go b/src/k8s/pkg/k8sd/pki/load.go new file mode 100644 index 000000000..243406f3a --- /dev/null +++ b/src/k8s/pkg/k8sd/pki/load.go @@ -0,0 +1,32 @@ +package pki + +import ( + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "fmt" +) + +// loadCertificate parses the PEM blocks and returns the certificate and private key. +// loadCertificate will fail if certPEM is not a valid certificate. +// loadCertificate will return a nil private key if keyPEM is empty, but will fail if it is not valid. +func loadCertificate(certPEM string, keyPEM string) (*x509.Certificate, *rsa.PrivateKey, error) { + decodedCert, _ := pem.Decode([]byte(certPEM)) + if decodedCert == nil { + return nil, nil, fmt.Errorf("failed to parse certificate PEM") + } + cert, err := x509.ParseCertificate(decodedCert.Bytes) + if err != nil { + return nil, nil, fmt.Errorf("failed to parse certificate: %w", err) + } + + var key *rsa.PrivateKey + if keyPEM != "" { + pb, _ := pem.Decode([]byte(keyPEM)) + key, err = x509.ParsePKCS1PrivateKey(pb.Bytes) + if err != nil { + return nil, nil, fmt.Errorf("failed to parse private key: %w", err) + } + } + return cert, key, nil +} diff --git a/src/k8s/pkg/k8sd/pki/worker.go b/src/k8s/pkg/k8sd/pki/worker.go new file mode 100644 index 000000000..683c65b39 --- /dev/null +++ b/src/k8s/pkg/k8sd/pki/worker.go @@ -0,0 +1,51 @@ +package pki + +import ( + "crypto/x509/pkix" + "fmt" + "net" +) + +type WorkerNodePKI struct { + CACert string + KubeletCert string + KubeletKey string +} + +// CompleteWorkerNodePKI generates the PKI needed for a worker node. +func (c *ControlPlanePKI) CompleteWorkerNodePKI(hostname string, nodeIP net.IP, bits int) (*WorkerNodePKI, error) { + caCert, caKey, err := loadCertificate(c.CACert, c.CAKey) + if err != nil { + return nil, fmt.Errorf("failed to load kubernetes CA: %w", err) + } + + // we do not have a CA key to sign the kubelet certificate, only send the cluster CA + if caKey == nil { + return &WorkerNodePKI{CACert: c.CACert}, nil + } + + template, err := generateCertificate(pkix.Name{CommonName: fmt.Sprintf("system:node:%s", hostname), Organization: []string{"system:nodes"}}, c.years, false, []string{hostname}, []net.IP{{127, 0, 0, 1}, nodeIP}) + if err != nil { + return nil, fmt.Errorf("failed to generate kubelet certificate for hostname=%s address=%s: %w", hostname, nodeIP.String(), err) + } + cert, key, err := signCertificate(template, bits, caCert, &caKey.PublicKey, caKey) + if err != nil { + return nil, fmt.Errorf("failed to sign kubelet certificate for hostname=%s address=%s: %w", hostname, nodeIP.String(), err) + } + + return &WorkerNodePKI{ + CACert: c.CACert, + KubeletCert: cert, + KubeletKey: key, + }, nil +} + +func (c *WorkerNodePKI) CompleteCertificates() error { + if c.CACert == "" { + return fmt.Errorf("kubernetes CA not specified") + } + if c.KubeletCert == "" || c.KubeletKey == "" { + return fmt.Errorf("kubelet certificate not specified") + } + return nil +} diff --git a/src/k8s/pkg/k8sd/setup/certificates.go b/src/k8s/pkg/k8sd/setup/certificates.go new file mode 100644 index 000000000..60008eceb --- /dev/null +++ b/src/k8s/pkg/k8sd/setup/certificates.go @@ -0,0 +1,76 @@ +package setup + +import ( + "fmt" + "os" + "path" + + "github.com/canonical/k8s/pkg/k8sd/pki" + "github.com/canonical/k8s/pkg/snap" +) + +func EnsureControlPlanePKI(snap snap.Snap, certificates *pki.ControlPlanePKI) error { + toWrite := map[string]string{ + path.Join(snap.KubernetesPKIDir(), "ca.crt"): certificates.CACert, + path.Join(snap.KubernetesPKIDir(), "front-proxy-ca.crt"): certificates.FrontProxyCACert, + path.Join(snap.KubernetesPKIDir(), "front-proxy-client.crt"): certificates.FrontProxyClientCert, + path.Join(snap.KubernetesPKIDir(), "front-proxy-client.key"): certificates.FrontProxyClientKey, + path.Join(snap.KubernetesPKIDir(), "apiserver.crt"): certificates.APIServerCert, + path.Join(snap.KubernetesPKIDir(), "apiserver.key"): certificates.APIServerKey, + path.Join(snap.KubernetesPKIDir(), "apiserver-kubelet-client.crt"): certificates.APIServerKubeletClientCert, + path.Join(snap.KubernetesPKIDir(), "apiserver-kubelet-client.key"): certificates.APIServerKubeletClientKey, + path.Join(snap.KubernetesPKIDir(), "kubelet.crt"): certificates.KubeletCert, + path.Join(snap.KubernetesPKIDir(), "kubelet.key"): certificates.KubeletKey, + path.Join(snap.KubernetesPKIDir(), "serviceaccount.key"): certificates.ServiceAccountKey, + path.Join(snap.K8sDqliteStateDir(), "cluster.crt"): certificates.K8sDqliteCert, + path.Join(snap.K8sDqliteStateDir(), "cluster.key"): certificates.K8sDqliteKey, + } + + if certificates.CAKey != "" { + toWrite[path.Join(snap.KubernetesPKIDir(), "ca.key")] = certificates.CAKey + } + if certificates.FrontProxyCAKey != "" { + toWrite[path.Join(snap.KubernetesPKIDir(), "front-proxy-ca.key")] = certificates.FrontProxyCACert + } + + for fname, cert := range toWrite { + if err := os.WriteFile(fname, []byte(cert), 0600); err != nil { + return fmt.Errorf("failed to write %s: %w", path.Base(fname), err) + } + if err := os.Chown(fname, snap.UID(), snap.GID()); err != nil { + return fmt.Errorf("failed to chown %s: %w", fname, err) + } + if err := os.Chmod(fname, 0600); err != nil { + return fmt.Errorf("failed to chmod %s: %w", fname, err) + } + } + + return nil +} + +func EnsureWorkerPKI(snap snap.Snap, certificates *pki.WorkerNodePKI) error { + toWrite := map[string]string{ + path.Join(snap.KubernetesPKIDir(), "ca.crt"): certificates.CACert, + } + + if certificates.KubeletCert != "" { + toWrite[path.Join(snap.KubernetesPKIDir(), "kubelet.crt")] = certificates.KubeletCert + } + if certificates.KubeletKey != "" { + toWrite[path.Join(snap.KubernetesPKIDir(), "kubelet.key")] = certificates.KubeletKey + } + + for fname, cert := range toWrite { + if err := os.WriteFile(fname, []byte(cert), 0600); err != nil { + return fmt.Errorf("failed to write %s: %w", path.Base(fname), err) + } + if err := os.Chown(fname, snap.UID(), snap.GID()); err != nil { + return fmt.Errorf("failed to chown %s: %w", fname, err) + } + if err := os.Chmod(fname, 0600); err != nil { + return fmt.Errorf("failed to chmod %s: %w", fname, err) + } + } + + return nil +} diff --git a/src/k8s/pkg/k8sd/setup/containerd.go b/src/k8s/pkg/k8sd/setup/containerd.go new file mode 100644 index 000000000..c1279aa55 --- /dev/null +++ b/src/k8s/pkg/k8sd/setup/containerd.go @@ -0,0 +1,69 @@ +package setup + +import ( + _ "embed" + "fmt" + "os" + "path" + + "github.com/canonical/k8s/pkg/snap" + snaputil "github.com/canonical/k8s/pkg/snap/util" + "github.com/canonical/k8s/pkg/utils" +) + +var ( + containerdConfigTomlTemplate = mustTemplate("containerd", "config.toml") +) + +type containerdConfigTomlConfig struct { + CNIConfDir string + CNIBinDir string + ImportsDir string + RegistryConfigDir string + PauseImage string +} + +// Containerd configures configuration and arguments for containerd on the local node. +func Containerd(snap snap.Snap) error { + configToml, err := os.OpenFile(path.Join(snap.ContainerdConfigDir(), "config.toml"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + return fmt.Errorf("failed to open config.toml: %w", err) + } + defer configToml.Close() + if err := containerdConfigTomlTemplate.Execute(configToml, containerdConfigTomlConfig{ + CNIConfDir: snap.CNIConfDir(), + CNIBinDir: snap.CNIBinDir(), + ImportsDir: snap.ContainerdExtraConfigDir(), + RegistryConfigDir: snap.ContainerdRegistryConfigDir(), + PauseImage: "registry.k8s.io/pause:3.7", + }); err != nil { + return fmt.Errorf("failed to write config.toml: %w", err) + } + + if _, err := snaputil.UpdateServiceArguments(snap, "containerd", map[string]string{ + "--config": path.Join(snap.ContainerdConfigDir(), "config.toml"), + "--address": path.Join(snap.ContainerdSocketDir(), "containerd.sock"), + "--root": snap.ContainerdRootDir(), + "--state": snap.ContainerdStateDir(), + }, nil); err != nil { + return fmt.Errorf("failed to write arguments file: %w", err) + } + + cniBinary := path.Join(snap.CNIBinDir(), "cni") + if err := utils.CopyFile(snap.CNIPluginsBinary(), cniBinary); err != nil { + return fmt.Errorf("failed to copy cni plugin binary: %w", err) + } + if err := os.Chmod(cniBinary, 0700); err != nil { + return fmt.Errorf("failed to chmod cni plugin binary: %w", err) + } + if err := os.Chown(cniBinary, snap.UID(), snap.GID()); err != nil { + return fmt.Errorf("failed to chown cni plugin binary: %w", err) + } + for _, plugin := range snap.CNIPlugins() { + if err := os.Symlink("cni", path.Join(snap.CNIBinDir(), plugin)); err != nil { + return fmt.Errorf("failed to symlink cni plugin %s: %w", plugin, err) + } + } + + return nil +} diff --git a/src/k8s/pkg/k8sd/setup/containerd_test.go b/src/k8s/pkg/k8sd/setup/containerd_test.go new file mode 100644 index 000000000..787f5f1cc --- /dev/null +++ b/src/k8s/pkg/k8sd/setup/containerd_test.go @@ -0,0 +1,106 @@ +package setup_test + +import ( + "fmt" + "io/fs" + "os" + "path" + "syscall" + "testing" + + "github.com/canonical/k8s/pkg/k8sd/setup" + "github.com/canonical/k8s/pkg/snap/mock" + snaputil "github.com/canonical/k8s/pkg/snap/util" + . "github.com/onsi/gomega" +) + +func TestContainerd(t *testing.T) { + g := NewWithT(t) + + dir := t.TempDir() + + g.Expect(os.WriteFile(path.Join(dir, "mockcni"), []byte("echo hi"), 0600)).To(BeNil()) + + s := &mock.Snap{ + Mock: mock.Mock{ + ContainerdConfigDir: path.Join(dir, "containerd"), + ContainerdRootDir: path.Join(dir, "containerd-root"), + ContainerdSocketDir: path.Join(dir, "containerd-run"), + ContainerdRegistryConfigDir: path.Join(dir, "containerd-registries"), + ContainerdStateDir: path.Join(dir, "containerd-state"), + ContainerdExtraConfigDir: path.Join(dir, "containerd-confd"), + ServiceArgumentsDir: path.Join(dir, "args"), + CNIBinDir: path.Join(dir, "opt-cni-bin"), + CNIConfDir: path.Join(dir, "cni-netd"), + CNIPluginsBinary: path.Join(dir, "mockcni"), + CNIPlugins: []string{"plugin1", "plugin2"}, + UID: os.Getuid(), + GID: os.Getgid(), + }, + } + + g.Expect(setup.EnsureAllDirectories(s)).To(BeNil()) + g.Expect(setup.Containerd(s)).To(BeNil()) + + t.Run("Config", func(t *testing.T) { + g := NewWithT(t) + b, err := os.ReadFile(path.Join(dir, "containerd", "config.toml")) + g.Expect(err).To(BeNil()) + g.Expect(string(b)).To(SatisfyAll( + ContainSubstring(fmt.Sprintf(`imports = ["%s/*.toml"]`, path.Join(dir, "containerd-confd"))), + ContainSubstring(fmt.Sprintf(`conf_dir = "%s"`, path.Join(dir, "cni-netd"))), + ContainSubstring(fmt.Sprintf(`bin_dir = "%s"`, path.Join(dir, "opt-cni-bin"))), + ContainSubstring(fmt.Sprintf(`config_path = "%s"`, path.Join(dir, "containerd-registries"))), + )) + + info, err := os.Stat(path.Join(dir, "containerd", "config.toml")) + g.Expect(err).To(BeNil()) + g.Expect(info.Mode().Perm()).To(Equal(fs.FileMode(0600))) + + switch stat := info.Sys().(type) { + case *syscall.Stat_t: + g.Expect(stat.Uid).To(Equal(uint32(os.Getuid()))) + g.Expect(stat.Gid).To(Equal(uint32(os.Getgid()))) + default: + g.Fail("failed to stat config.toml") + } + }) + + t.Run("CNI", func(t *testing.T) { + g := NewWithT(t) + for _, plugin := range []string{"plugin1", "plugin2"} { + link, err := os.Readlink(path.Join(dir, "opt-cni-bin", plugin)) + g.Expect(err).To(BeNil()) + g.Expect(link).To(Equal("cni")) + } + + info, err := os.Stat(path.Join(dir, "opt-cni-bin")) + g.Expect(err).To(BeNil()) + g.Expect(info.Mode().Perm()).To(Equal(fs.FileMode(0700))) + + switch stat := info.Sys().(type) { + case *syscall.Stat_t: + g.Expect(stat.Uid).To(Equal(uint32(os.Getuid()))) + g.Expect(stat.Gid).To(Equal(uint32(os.Getgid()))) + default: + g.Fail("failed to stat installed cni") + } + }) + + t.Run("Args", func(t *testing.T) { + for key, expectedVal := range map[string]string{ + "--config": path.Join(dir, "containerd", "config.toml"), + "--state": path.Join(dir, "containerd-state"), + "--root": path.Join(dir, "containerd-root"), + "--address": path.Join(dir, "containerd-run", "containerd.sock"), + } { + t.Run(key, func(t *testing.T) { + g := NewWithT(t) + val, err := snaputil.GetServiceArgument(s, "containerd", key) + g.Expect(err).To(BeNil()) + g.Expect(val).To(Equal(expectedVal)) + }) + } + }) + +} diff --git a/src/k8s/pkg/k8sd/setup/directories.go b/src/k8s/pkg/k8sd/setup/directories.go new file mode 100644 index 000000000..33f45a7f5 --- /dev/null +++ b/src/k8s/pkg/k8sd/setup/directories.go @@ -0,0 +1,32 @@ +package setup + +import ( + "fmt" + "os" + + "github.com/canonical/k8s/pkg/snap" +) + +// EnsureAllDirectories ensures all required configuration and state directories are created. +func EnsureAllDirectories(snap snap.Snap) error { + for _, dir := range []string{ + snap.CNIBinDir(), + snap.CNIConfDir(), + snap.ContainerdConfigDir(), + snap.ContainerdExtraConfigDir(), + snap.ContainerdRegistryConfigDir(), + snap.K8sDqliteStateDir(), + snap.KubernetesConfigDir(), + snap.KubernetesPKIDir(), + snap.ServiceArgumentsDir(), + snap.ServiceExtraConfigDir(), + } { + if dir == "" { + continue + } + if err := os.MkdirAll(dir, 0700); err != nil { + return fmt.Errorf("failed to create required directory: %w", err) + } + } + return nil +} diff --git a/src/k8s/pkg/k8sd/setup/embed/apiserver/auth-token-webhook.conf b/src/k8s/pkg/k8sd/setup/embed/apiserver/auth-token-webhook.conf new file mode 100644 index 000000000..e32f8c33e --- /dev/null +++ b/src/k8s/pkg/k8sd/setup/embed/apiserver/auth-token-webhook.conf @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Config +clusters: + - name: k8s-token-auth-service + cluster: + insecure-skip-tls-verify: true + server: "{{ .URL }}" +current-context: webhook +contexts: +- context: + cluster: k8s-token-auth-service + user: k8s-apiserver + name: webhook +users: + - name: k8s-apiserver + user: {} diff --git a/src/k8s/pkg/k8sd/setup/embed/containerd/config.toml b/src/k8s/pkg/k8sd/setup/embed/containerd/config.toml new file mode 100644 index 000000000..0c06adaba --- /dev/null +++ b/src/k8s/pkg/k8sd/setup/embed/containerd/config.toml @@ -0,0 +1,46 @@ +version = 2 +oom_score = 0 + +imports = ["{{ .ImportsDir }}/*.toml"] + +[grpc] + uid = 0 + gid = 0 + max_recv_message_size = 16777216 + max_send_message_size = 16777216 + +[debug] + address = "" + uid = 0 + gid = 0 + level = "" + +[metrics] + address = "" + grpc_histogram = false + +[cgroup] + path = "" + +[plugins."io.containerd.grpc.v1.cri"] + stream_server_address = "127.0.0.1" + stream_server_port = "0" + enable_selinux = false + sandbox_image = "{{ .PauseImage }}" + stats_collect_period = 10 + enable_tls_streaming = false + max_container_log_line_size = 16384 + + [plugins."io.containerd.grpc.v1.cri".containerd] + no_pivot = false + default_runtime_name = "runc" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".cni] + bin_dir = "{{ .CNIBinDir }}" + conf_dir = "{{ .CNIConfDir }}" + + [plugins."io.containerd.grpc.v1.cri".registry] + config_path = "{{ .RegistryConfigDir }}" diff --git a/src/k8s/pkg/k8sd/setup/embed/kubeconfig b/src/k8s/pkg/k8sd/setup/embed/kubeconfig new file mode 100644 index 000000000..e9350c34b --- /dev/null +++ b/src/k8s/pkg/k8sd/setup/embed/kubeconfig @@ -0,0 +1,17 @@ +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: {{ .CA }} + server: https://{{ .URL }} + name: k8s +contexts: +- context: + cluster: k8s + user: k8s-user + name: k8s +current-context: k8s +kind: Config +users: +- name: k8s-user + user: + token: {{ .Token }} diff --git a/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go b/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go new file mode 100644 index 000000000..8ba864a77 --- /dev/null +++ b/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go @@ -0,0 +1,28 @@ +package setup + +import ( + "fmt" + "path" + + "github.com/canonical/k8s/pkg/proxy" + "github.com/canonical/k8s/pkg/snap" + snaputil "github.com/canonical/k8s/pkg/snap/util" +) + +// K8sAPIServerProxy prepares configuration for k8s-apiserver-proxy. +func K8sAPIServerProxy(snap snap.Snap, servers []string) error { + configFile := path.Join(snap.ServiceExtraConfigDir(), "k8s-apiserver-proxy.json") + if err := proxy.WriteEndpointsConfig(servers, configFile); err != nil { + return fmt.Errorf("failed to write proxy configuration file: %w", err) + } + + if _, err := snaputil.UpdateServiceArguments(snap, "k8s-apiserver-proxy", map[string]string{ + "--listen": "127.0.0.1:6443", + "--kubeconfig": path.Join(snap.KubernetesConfigDir(), "kubelet.conf"), + "--endpoints": configFile, + }, nil); err != nil { + return fmt.Errorf("failed to write arguments file: %w", err) + } + + return nil +} diff --git a/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy_test.go b/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy_test.go new file mode 100644 index 000000000..1f16475a1 --- /dev/null +++ b/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy_test.go @@ -0,0 +1,8 @@ +package setup_test + +import "testing" + +func TestK8sAPIServerProxy(t *testing.T) { + // TODO: setup a mock snap with paths, then verify setup.K8sAPIServerProxy(). + // See TestContainerd() for inspiration. +} diff --git a/src/k8s/pkg/k8sd/setup/k8s_dqlite.go b/src/k8s/pkg/k8sd/setup/k8s_dqlite.go new file mode 100644 index 000000000..a9f72c35b --- /dev/null +++ b/src/k8s/pkg/k8sd/setup/k8s_dqlite.go @@ -0,0 +1,35 @@ +package setup + +import ( + "fmt" + "os" + "path" + + "github.com/canonical/k8s/pkg/snap" + snaputil "github.com/canonical/k8s/pkg/snap/util" + "gopkg.in/yaml.v2" +) + +type k8sDqliteInit struct { + Address string `yaml:"Address,omitempty"` + Cluster []string `yaml:"Cluster,omitempty"` +} + +func K8sDqlite(snap snap.Snap, address string, cluster []string) error { + b, err := yaml.Marshal(&k8sDqliteInit{Address: address, Cluster: cluster}) + if err != nil { + return fmt.Errorf("failed to create init.yaml file for address=%s cluster=%v: %w", address, cluster, err) + } + + if err := os.WriteFile(path.Join(snap.K8sDqliteStateDir(), "init.yaml"), b, 0600); err != nil { + return fmt.Errorf("failed to write init.yaml: %w", err) + } + + if _, err := snaputil.UpdateServiceArguments(snap, "k8s-dqlite", map[string]string{ + "--storage-dir": snap.K8sDqliteStateDir(), + "--listen": fmt.Sprintf("unix://%s", path.Join(snap.K8sDqliteStateDir(), "k8s-dqlite.sock")), + }, nil); err != nil { + return fmt.Errorf("failed to write arguments file: %w", err) + } + return nil +} diff --git a/src/k8s/pkg/k8sd/setup/kube_apiserver.go b/src/k8s/pkg/k8sd/setup/kube_apiserver.go new file mode 100644 index 000000000..04f90630e --- /dev/null +++ b/src/k8s/pkg/k8sd/setup/kube_apiserver.go @@ -0,0 +1,71 @@ +package setup + +import ( + "fmt" + "os" + "path" + + "github.com/canonical/k8s/pkg/snap" + snaputil "github.com/canonical/k8s/pkg/snap/util" +) + +var apiserverAuthTokenWebhookTemplate = mustTemplate("apiserver", "auth-token-webhook.conf") + +type apiserverAuthTokenWebhookTemplateConfig struct { + URL string +} + +// KubeAPIServer configures kube-apiserver on the local node. +func KubeAPIServer(snap snap.Snap, serviceCIDR string, authWebhookURL string, enableFrontProxy bool, datastore string, authorizationMode string) error { + authTokenWebhookConfigFile := path.Join(snap.ServiceExtraConfigDir(), "auth-token-webhook.conf") + authTokenWebhookFile, err := os.OpenFile(authTokenWebhookConfigFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + return fmt.Errorf("failed to open auth-token-webhook.conf: %w", err) + } + if err := apiserverAuthTokenWebhookTemplate.Execute(authTokenWebhookFile, apiserverAuthTokenWebhookTemplateConfig{ + URL: authWebhookURL, + }); err != nil { + return fmt.Errorf("failed to write auth-token-webhook.conf: %w", err) + } + defer authTokenWebhookFile.Close() + + args := map[string]string{ + "--service-cluster-ip-range": serviceCIDR, + "--authorization-mode": authorizationMode, + "--service-account-key-file": path.Join(snap.KubernetesPKIDir(), "serviceaccount.key"), + "--service-account-signing-key-file": path.Join(snap.KubernetesPKIDir(), "serviceaccount.key"), + "--client-ca-file": path.Join(snap.KubernetesPKIDir(), "ca.crt"), + "--tls-cert-file": path.Join(snap.KubernetesPKIDir(), "apiserver.crt"), + "--tls-private-key-file": path.Join(snap.KubernetesPKIDir(), "apiserver.key"), + "--tls-cipher-suites": "TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384", + "--kubelet-client-certificate": path.Join(snap.KubernetesPKIDir(), "apiserver-kubelet-client.crt"), + "--kubelet-client-key": path.Join(snap.KubernetesPKIDir(), "apiserver-kubelet-client.key"), + "--secure-port": "6443", + "--allow-privileged": "true", + "--service-account-issuer": "https://kubernetes.default.svc", + "--authentication-token-webhook-config-file": authTokenWebhookConfigFile, + "--kubelet-preferred-address-types": "InternalIP,Hostname,InternalDNS,ExternalDNS,ExternalIP", + "--kubelet-certificate-authority": path.Join(snap.KubernetesPKIDir(), "ca.crt"), + } + + switch datastore { + case "k8s-dqlite": + args["--etcd-servers"] = fmt.Sprintf("unix://%s", path.Join(snap.K8sDqliteStateDir(), "k8s-dqlite.sock")) + default: + return fmt.Errorf("unsupported datastore %s. must be 'k8s-dqlite'", datastore) + } + + if enableFrontProxy { + args["--requestheader-client-ca-file"] = path.Join(snap.KubernetesPKIDir(), "front-proxy-ca.crt") + args["--requestheader-allowed-names"] = "front-proxy-client" + args["--requestheader-extra-headers-prefix"] = "X-Remote-Extra-" + args["--requestheader-group-headers"] = "X-Remote-Group" + args["--requestheader-username-headers"] = "X-Remote-User" + args["--proxy-client-cert-file"] = path.Join(snap.KubernetesPKIDir(), "front-proxy-client.crt") + args["--proxy-client-key-file"] = path.Join(snap.KubernetesPKIDir(), "front-proxy-client.key") + } + if _, err := snaputil.UpdateServiceArguments(snap, "kube-apiserver", args, nil); err != nil { + return fmt.Errorf("failed to render arguments file: %w", err) + } + return nil +} diff --git a/src/k8s/pkg/k8sd/setup/kube_apiserver_test.go b/src/k8s/pkg/k8sd/setup/kube_apiserver_test.go new file mode 100644 index 000000000..681c68c7f --- /dev/null +++ b/src/k8s/pkg/k8sd/setup/kube_apiserver_test.go @@ -0,0 +1,32 @@ +package setup_test + +import ( + "os" + "path" + "testing" + + "github.com/canonical/k8s/pkg/k8sd/setup" + "github.com/canonical/k8s/pkg/snap/mock" + . "github.com/onsi/gomega" +) + +func TestKubeAPIServer(t *testing.T) { + g := NewWithT(t) + + dir := t.TempDir() + + s := &mock.Snap{ + Mock: mock.Mock{ + UID: os.Getuid(), + GID: os.Getgid(), + KubernetesConfigDir: path.Join(dir, "kubernetes"), + KubernetesPKIDir: path.Join(dir, "kubernetes-pki"), + ServiceArgumentsDir: path.Join(dir, "args"), + ServiceExtraConfigDir: path.Join(dir, "args/conf.d"), + K8sDqliteStateDir: path.Join(dir, "k8s-dqlite"), + }, + } + + g.Expect(setup.EnsureAllDirectories(s)).To(BeNil()) + g.Expect(setup.KubeAPIServer(s, "10.152.0.0/16", "https://10.0.0.1:6400/1.0/kubernetes/auth/webhook", false, "k8s-dqlite", "Node,RBAC")).To(BeNil()) +} diff --git a/src/k8s/pkg/k8sd/setup/kube_controller_manager.go b/src/k8s/pkg/k8sd/setup/kube_controller_manager.go new file mode 100644 index 000000000..fc5c8c398 --- /dev/null +++ b/src/k8s/pkg/k8sd/setup/kube_controller_manager.go @@ -0,0 +1,34 @@ +package setup + +import ( + "fmt" + "os" + "path" + + "github.com/canonical/k8s/pkg/snap" + snaputil "github.com/canonical/k8s/pkg/snap/util" +) + +// KubeControllerManager configures kube-controller-manager on the local node. +func KubeControllerManager(snap snap.Snap) error { + args := map[string]string{ + "--kubeconfig": path.Join(snap.KubernetesConfigDir(), "controller.conf"), + "--authorization-kubeconfig": path.Join(snap.KubernetesConfigDir(), "controller.conf"), + "--authentication-kubeconfig": path.Join(snap.KubernetesConfigDir(), "controller.conf"), + "--service-account-private-key-file": path.Join(snap.KubernetesPKIDir(), "serviceaccount.key"), + "--root-ca-file": path.Join(snap.KubernetesPKIDir(), "ca.crt"), + "--use-service-account-credentials": "true", + "--profiling": "false", + "--leader-elect-lease-duration": "30s", + "--leader-elect-renew-deadline": "15s", + } + // enable cluster-signing if certificates are available + if _, err := os.Stat(path.Join(snap.KubernetesPKIDir(), "ca.key")); err == nil { + args["--cluster-signing-cert-file"] = path.Join(snap.KubernetesPKIDir(), "ca.crt") + args["--cluster-signing-key-file"] = path.Join(snap.KubernetesPKIDir(), "ca.key") + } + if _, err := snaputil.UpdateServiceArguments(snap, "kube-controller-manager", args, nil); err != nil { + return fmt.Errorf("failed to render arguments file: %w", err) + } + return nil +} diff --git a/src/k8s/pkg/k8sd/setup/kube_proxy.go b/src/k8s/pkg/k8sd/setup/kube_proxy.go new file mode 100644 index 000000000..c462b9ced --- /dev/null +++ b/src/k8s/pkg/k8sd/setup/kube_proxy.go @@ -0,0 +1,23 @@ +package setup + +import ( + "fmt" + "path" + + "github.com/canonical/k8s/pkg/snap" + snaputil "github.com/canonical/k8s/pkg/snap/util" +) + +// KubeProxy configures kube-proxy on the local node. +func KubeProxy(snap snap.Snap, hostname string, podCIDR string) error { + if _, err := snaputil.UpdateServiceArguments(snap, "kube-proxy", map[string]string{ + "--kubeconfig": path.Join(snap.KubernetesConfigDir(), "proxy.conf"), + "--cluster-cidr": podCIDR, + "--healthz-bind-address": "127.0.0.1", + "--profiling": "false", + "--hostname-override": hostname, + }, nil); err != nil { + return fmt.Errorf("failed to render arguments file: %w", err) + } + return nil +} diff --git a/src/k8s/pkg/k8sd/setup/kube_scheduler.go b/src/k8s/pkg/k8sd/setup/kube_scheduler.go new file mode 100644 index 000000000..3d8586d5e --- /dev/null +++ b/src/k8s/pkg/k8sd/setup/kube_scheduler.go @@ -0,0 +1,24 @@ +package setup + +import ( + "fmt" + "path" + + "github.com/canonical/k8s/pkg/snap" + snaputil "github.com/canonical/k8s/pkg/snap/util" +) + +// KubeScheduler configures kube-scheduler on the local node. +func KubeScheduler(snap snap.Snap) error { + if _, err := snaputil.UpdateServiceArguments(snap, "kube-scheduler", map[string]string{ + "--kubeconfig": path.Join(snap.KubernetesConfigDir(), "scheduler.conf"), + "--authorization-kubeconfig": path.Join(snap.KubernetesConfigDir(), "scheduler.conf"), + "--authentication-kubeconfig": path.Join(snap.KubernetesConfigDir(), "scheduler.conf"), + "--profiling": "false", + "--leader-elect-lease-duration": "30s", + "--leader-elect-renew-deadline": "15s", + }, nil); err != nil { + return fmt.Errorf("failed to render arguments file: %w", err) + } + return nil +} diff --git a/src/k8s/pkg/k8sd/setup/kubelet.go b/src/k8s/pkg/k8sd/setup/kubelet.go new file mode 100644 index 000000000..29f009be9 --- /dev/null +++ b/src/k8s/pkg/k8sd/setup/kubelet.go @@ -0,0 +1,46 @@ +package setup + +import ( + "fmt" + "net" + "path" + + "github.com/canonical/k8s/pkg/snap" + snaputil "github.com/canonical/k8s/pkg/snap/util" +) + +// Kubelet configures kubelet on the local node. +func Kubelet(snap snap.Snap, hostname string, nodeIP net.IP, clusterDNS string, clusterDomain string, cloudProvider string) error { + args := map[string]string{ + "--kubeconfig": path.Join(snap.KubernetesConfigDir(), "kubelet.conf"), + "--client-ca-file": path.Join(snap.KubernetesPKIDir(), "ca.crt"), + "--cert-dir": snap.KubernetesPKIDir(), + "--root-dir": snap.KubeletRootDir(), + "--hostname-override": hostname, + "--anonymous-auth": "false", + "--fail-swap-on": "false", + "--eviction-hard": "'memory.available<100Mi,nodefs.available<1Gi,imagefs.available<1Gi'", + "--container-runtime-endpoint": path.Join(snap.ContainerdSocketDir(), "containerd.sock"), + "--containerd": path.Join(snap.ContainerdSocketDir(), "containerd.sock"), + "--authentication-token-webhook": "true", + "--read-only-port": "0", + "--tls-cipher-suites": "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256", + "--serialize-image-pulls": "false", + } + if cloudProvider != "" { + args["--cloud-provider"] = cloudProvider + } + if clusterDNS != "" { + args["--cluster-dns"] = clusterDNS + } + if clusterDomain != "" { + args["--cluster-domain"] = clusterDomain + } + if nodeIP != nil { + args["--node-ip"] = nodeIP.String() + } + if _, err := snaputil.UpdateServiceArguments(snap, "kubelet", args, nil); err != nil { + return fmt.Errorf("failed to render arguments file: %w", err) + } + return nil +} diff --git a/src/k8s/pkg/k8sd/setup/kubelet_test.go b/src/k8s/pkg/k8sd/setup/kubelet_test.go new file mode 100644 index 000000000..4e3b3e56e --- /dev/null +++ b/src/k8s/pkg/k8sd/setup/kubelet_test.go @@ -0,0 +1,8 @@ +package setup_test + +import "testing" + +func TestKubelet(t *testing.T) { + // TODO: setup a mock snap with paths, then verify setup.Kubelet(). + // See TestContainerd() for inspiration. +} diff --git a/src/k8s/pkg/k8sd/setup/templates.go b/src/k8s/pkg/k8sd/setup/templates.go new file mode 100644 index 000000000..8725c8f6a --- /dev/null +++ b/src/k8s/pkg/k8sd/setup/templates.go @@ -0,0 +1,22 @@ +package setup + +import ( + "embed" + "fmt" + "path/filepath" + "text/template" +) + +var ( + //go:embed embed + templates embed.FS +) + +func mustTemplate(parts ...string) *template.Template { + path := filepath.Join(append([]string{"embed"}, parts...)...) + b, err := templates.ReadFile(path) + if err != nil { + panic(fmt.Errorf("invalid template %s: %s", path, err)) + } + return template.Must(template.New(path).Parse(string(b))) +} diff --git a/src/k8s/pkg/k8sd/setup/util_kubeconfig.go b/src/k8s/pkg/k8sd/setup/util_kubeconfig.go new file mode 100644 index 000000000..10d6c9fdb --- /dev/null +++ b/src/k8s/pkg/k8sd/setup/util_kubeconfig.go @@ -0,0 +1,40 @@ +package setup + +import ( + "encoding/base64" + "fmt" + "io" + "os" +) + +var ( + kubeconfigTemplate = mustTemplate("kubeconfig") +) + +type kubeconfigTemplateConfig struct { + CA string + URL string + Token string +} + +// renderKubeconfig writes a kubeconfig to the specified writer. +func renderKubeconfig(writer io.Writer, token string, url string, caPEM string) error { + if err := kubeconfigTemplate.Execute(writer, kubeconfigTemplateConfig{ + CA: base64.StdEncoding.EncodeToString([]byte(caPEM)), + URL: url, + Token: token, + }); err != nil { + return fmt.Errorf("failed to render template: %w", err) + } + return nil +} + +// Kubeconfig writes a kubeconfig file to disk. +func Kubeconfig(path string, token string, url string, caPEM string) error { + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + return fmt.Errorf("failed to open file: %w", err) + } + defer file.Close() + return renderKubeconfig(file, token, url, caPEM) +} diff --git a/src/k8s/pkg/k8sd/types/cluster_config.go b/src/k8s/pkg/k8sd/types/cluster_config.go index f1d389956..6d23a99d0 100644 --- a/src/k8s/pkg/k8sd/types/cluster_config.go +++ b/src/k8s/pkg/k8sd/types/cluster_config.go @@ -5,26 +5,27 @@ import apiv1 "github.com/canonical/k8s/api/v1" // ClusterConfig is the control plane configuration format of the k8s cluster. // ClusterConfig should attempt to use structured fields wherever possible. type ClusterConfig struct { - Cluster Cluster `yaml:"cluster"` + Network Network `yaml:"network"` Certificates Certificates `yaml:"certificates"` Kubelet Kubelet `yaml:"kubelet"` K8sDqlite K8sDqlite `yaml:"k8s-dqlite"` APIServer APIServer `yaml:"apiserver"` } -type Cluster struct { - CIDR string `yaml:"cidr,omitempty"` +type Network struct { + PodCIDR string `yaml:"pod-cidr,omitempty"` + ServiceCIDR string `yaml:"svc-cidr,omitempty"` } type Certificates struct { - CACert string `yaml:"ca-crt,omitempty"` - CAKey string `yaml:"ca-key,omitempty"` - APIServerToKubeletCert string `yaml:"apiserver-to-kubelet-crt,omitempty"` - APIServerToKubeletKey string `yaml:"apiserver-to-kubelet-key,omitempty"` - K8sDqliteCert string `yaml:"k8s-dqlite-crt,omitempty"` - K8sDqliteKey string `yaml:"k8s-dqlite-key,omitempty"` - FrontProxyCACert string `yaml:"front-proxy-ca-crt,omitempty"` - FrontProxyCAKey string `yaml:"front-proxy-ca-key,omitempty"` + CACert string `yaml:"ca-crt,omitempty"` + CAKey string `yaml:"ca-key,omitempty"` + APIServerKubeletClientCert string `yaml:"apiserver-kubelet-client-crt,omitempty"` + APIServerKubeletClientKey string `yaml:"apiserver-kubelet-client-key,omitempty"` + K8sDqliteCert string `yaml:"k8s-dqlite-crt,omitempty"` + K8sDqliteKey string `yaml:"k8s-dqlite-key,omitempty"` + FrontProxyCACert string `yaml:"front-proxy-ca-crt,omitempty"` + FrontProxyCAKey string `yaml:"front-proxy-ca-key,omitempty"` } type Kubelet struct { @@ -50,10 +51,12 @@ type K8sDqlite struct { func DefaultClusterConfig() ClusterConfig { return ClusterConfig{ - Cluster: Cluster{ - CIDR: "10.1.0.0/16", + Network: Network{ + PodCIDR: "10.1.0.0/16", + ServiceCIDR: "10.152.183.0/24", }, APIServer: APIServer{ + Datastore: "k8s-dqlite", SecurePort: 6443, AuthorizationMode: "Node,RBAC", }, @@ -67,8 +70,8 @@ func DefaultClusterConfig() ClusterConfig { // and maps them to a ClusterConfig. func ClusterConfigFromBootstrapConfig(b *apiv1.BootstrapConfig) ClusterConfig { return ClusterConfig{ - Cluster: Cluster{ - CIDR: b.ClusterCIDR, + Network: Network{ + PodCIDR: b.ClusterCIDR, }, } } diff --git a/src/k8s/pkg/k8sd/types/cluster_config_merge.go b/src/k8s/pkg/k8sd/types/cluster_config_merge.go index 76927d4b9..f682315fd 100644 --- a/src/k8s/pkg/k8sd/types/cluster_config_merge.go +++ b/src/k8s/pkg/k8sd/types/cluster_config_merge.go @@ -33,13 +33,14 @@ func MergeClusterConfig(existing ClusterConfig, new ClusterConfig) (ClusterConfi {name: "cluster CA key", val: &config.Certificates.CAKey, old: existing.Certificates.CAKey, new: new.Certificates.CAKey}, {name: "k8s-dqlite certificate", val: &config.Certificates.K8sDqliteCert, old: existing.Certificates.K8sDqliteCert, new: new.Certificates.K8sDqliteCert}, {name: "k8s-dqlite key", val: &config.Certificates.K8sDqliteKey, old: existing.Certificates.K8sDqliteKey, new: new.Certificates.K8sDqliteKey}, - {name: "apiserver-to-kubelet certificate", val: &config.Certificates.APIServerToKubeletCert, old: existing.Certificates.APIServerToKubeletCert, new: new.Certificates.APIServerToKubeletCert, allowChange: true}, - {name: "apiserver-to-kubelet key", val: &config.Certificates.APIServerToKubeletKey, old: existing.Certificates.APIServerToKubeletKey, new: new.Certificates.APIServerToKubeletKey, allowChange: true}, + {name: "apiserver-kubelet-client certificate", val: &config.Certificates.APIServerKubeletClientCert, old: existing.Certificates.APIServerKubeletClientCert, new: new.Certificates.APIServerKubeletClientCert, allowChange: true}, + {name: "apiserver-kubelet-client key", val: &config.Certificates.APIServerKubeletClientKey, old: existing.Certificates.APIServerKubeletClientKey, new: new.Certificates.APIServerKubeletClientKey, allowChange: true}, {name: "front proxy CA certificate", val: &config.Certificates.FrontProxyCACert, old: existing.Certificates.FrontProxyCACert, new: new.Certificates.FrontProxyCACert, allowChange: true}, {name: "front proxy CA key", val: &config.Certificates.FrontProxyCAKey, old: existing.Certificates.FrontProxyCAKey, new: new.Certificates.FrontProxyCAKey, allowChange: true}, {name: "authorization-mode", val: &config.APIServer.AuthorizationMode, old: existing.APIServer.AuthorizationMode, new: new.APIServer.AuthorizationMode, allowChange: true}, {name: "service account key", val: &config.APIServer.ServiceAccountKey, old: existing.APIServer.ServiceAccountKey, new: new.APIServer.ServiceAccountKey}, - {name: "cluster cidr", val: &config.Cluster.CIDR, old: existing.Cluster.CIDR, new: new.Cluster.CIDR}, + {name: "pod cidr", val: &config.Network.PodCIDR, old: existing.Network.PodCIDR, new: new.Network.PodCIDR}, + {name: "service cidr", val: &config.Network.ServiceCIDR, old: existing.Network.ServiceCIDR, new: new.Network.ServiceCIDR}, {name: "datastore", val: &config.APIServer.Datastore, old: existing.APIServer.Datastore, new: new.APIServer.Datastore}, {name: "datastore url", val: &config.APIServer.DatastoreURL, old: existing.APIServer.DatastoreURL, new: new.APIServer.DatastoreURL, allowChange: true}, {name: "datastore ca", val: &config.APIServer.DatastoreCA, old: existing.APIServer.DatastoreCA, new: new.APIServer.DatastoreCA, allowChange: true}, diff --git a/src/k8s/pkg/k8sd/types/cluster_config_test.go b/src/k8s/pkg/k8sd/types/cluster_config_test.go index 911f04b30..ad07ba849 100644 --- a/src/k8s/pkg/k8sd/types/cluster_config_test.go +++ b/src/k8s/pkg/k8sd/types/cluster_config_test.go @@ -17,8 +17,8 @@ func TestClusterConfigFromBootstrapConfig(t *testing.T) { } expectedConfig := types.ClusterConfig{ - Cluster: types.Cluster{ - CIDR: "10.1.0.0/16", + Network: types.Network{ + PodCIDR: "10.1.0.0/16", }, } @@ -67,13 +67,14 @@ func TestMergeClusterConfig(t *testing.T) { generateMergeClusterConfigTestCases("CAKey", false, "v1", "v2", func(c *types.ClusterConfig, v any) { c.Certificates.CAKey = v.(string) }), generateMergeClusterConfigTestCases("K8sDqliteCert", false, "v1", "v2", func(c *types.ClusterConfig, v any) { c.Certificates.K8sDqliteCert = v.(string) }), generateMergeClusterConfigTestCases("K8sDqliteKey", false, "v1", "v2", func(c *types.ClusterConfig, v any) { c.Certificates.K8sDqliteKey = v.(string) }), - generateMergeClusterConfigTestCases("APIServerToKubeletCert", true, "v1", "v2", func(c *types.ClusterConfig, v any) { c.Certificates.APIServerToKubeletCert = v.(string) }), - generateMergeClusterConfigTestCases("APIServerToKubeletKey", true, "v1", "v2", func(c *types.ClusterConfig, v any) { c.Certificates.APIServerToKubeletKey = v.(string) }), + generateMergeClusterConfigTestCases("APIServerKubeletClientCert", true, "v1", "v2", func(c *types.ClusterConfig, v any) { c.Certificates.APIServerKubeletClientCert = v.(string) }), + generateMergeClusterConfigTestCases("APIServerKubeletClientKey", true, "v1", "v2", func(c *types.ClusterConfig, v any) { c.Certificates.APIServerKubeletClientKey = v.(string) }), generateMergeClusterConfigTestCases("FrontProxyCACert", true, "v1", "v2", func(c *types.ClusterConfig, v any) { c.Certificates.FrontProxyCACert = v.(string) }), generateMergeClusterConfigTestCases("FrontProxyCAKey", true, "v1", "v2", func(c *types.ClusterConfig, v any) { c.Certificates.FrontProxyCAKey = v.(string) }), generateMergeClusterConfigTestCases("AuthorizationMode", true, "v1", "v2", func(c *types.ClusterConfig, v any) { c.APIServer.AuthorizationMode = v.(string) }), generateMergeClusterConfigTestCases("ServiceAccountKey", false, "v1", "v2", func(c *types.ClusterConfig, v any) { c.APIServer.ServiceAccountKey = v.(string) }), - generateMergeClusterConfigTestCases("ClusterCIDR", false, "v1", "v2", func(c *types.ClusterConfig, v any) { c.Cluster.CIDR = v.(string) }), + generateMergeClusterConfigTestCases("PodCIDR", false, "v1", "v2", func(c *types.ClusterConfig, v any) { c.Network.PodCIDR = v.(string) }), + generateMergeClusterConfigTestCases("ServiceCIDR", false, "v1", "v2", func(c *types.ClusterConfig, v any) { c.Network.ServiceCIDR = v.(string) }), generateMergeClusterConfigTestCases("Datastore", false, "v1", "v2", func(c *types.ClusterConfig, v any) { c.APIServer.Datastore = v.(string) }), generateMergeClusterConfigTestCases("DatastoreURL", true, "v1", "v2", func(c *types.ClusterConfig, v any) { c.APIServer.DatastoreURL = v.(string) }), generateMergeClusterConfigTestCases("DatastoreCA", true, "v1", "v2", func(c *types.ClusterConfig, v any) { c.APIServer.DatastoreCA = v.(string) }), diff --git a/src/k8s/pkg/k8sd/types/component.go b/src/k8s/pkg/k8sd/types/component.go new file mode 100644 index 000000000..d7327d84c --- /dev/null +++ b/src/k8s/pkg/k8sd/types/component.go @@ -0,0 +1,13 @@ +package types + +// Component defines a Kubernetes component that can be deployed on the cluster. +type Component struct { + // DependsOn is a component that this component depends on. + DependsOn string + // ManifestPath is a path containing the manifests to deploy this component. + ManifestPath string + // ReleaseName is the name to use when applying this component on the cluster. + ReleaseName string + // Namespace is the namespace where this component is installed. + Namespace string +} diff --git a/src/k8s/pkg/snap/context.go b/src/k8s/pkg/snap/context.go new file mode 100644 index 000000000..ffd7d3253 --- /dev/null +++ b/src/k8s/pkg/snap/context.go @@ -0,0 +1,22 @@ +package snap + +import "context" + +type snapContextKey struct{} + +// SnapFromContext extracts the snap instance from the provided context. +// A panic is invoked if there is not snap instance in this context. +func SnapFromContext(ctx context.Context) Snap { + snap, ok := ctx.Value(snapContextKey{}).(Snap) + if !ok { + // This should never happen as the main microcluster state context should contain the snap for k8sd. + // Thus, panic is fine here to avoid cumbersome and unnecessary error checks on client side. + panic("There is no snap value in the given context. Make sure that the context is wrapped with snap.ContextWithSnap.") + } + return snap +} + +// ContextWithSnap adds a snap instance to a given context. +func ContextWithSnap(ctx context.Context, snap Snap) context.Context { + return context.WithValue(ctx, snapContextKey{}, snap) +} diff --git a/src/k8s/pkg/snap/interface.go b/src/k8s/pkg/snap/interface.go index 4e9509d0f..dffd727c4 100644 --- a/src/k8s/pkg/snap/interface.go +++ b/src/k8s/pkg/snap/interface.go @@ -1,27 +1,42 @@ package snap -import "context" +import ( + "context" -// Snap is how k8s interacts with the snap. + "github.com/canonical/k8s/pkg/k8sd/types" +) + +// Snap abstracts file system paths and interacting with the k8s services. type Snap interface { - // IsStrict returns true if the snap is installed with strict confinement. - IsStrict() bool - // ReadServiceArguments reads the arguments file for a particular service. - ReadServiceArguments(serviceName string) (string, error) - // WriteServiceArguments updates the arguments file a particular service. - WriteServiceArguments(serviceName string, b []byte) error - - // StartService starts a k8s service. - StartService(ctx context.Context, serviceName string) error - // StopService stops a k8s service. - StopService(ctx context.Context, serviceName string) error - // RestartService restarts a k8s service. - RestartService(ctx context.Context, serviceName string) error - - // Path concenates any passed path parts with the $SNAP path - Path(parts ...string) string - // DataPath concenates any passed path parts with the $SNAP_DATA path - DataPath(parts ...string) string - // CommonPath concenates any passed path parts with the $SNAP_COMMON path - CommonPath(parts ...string) string + Strict() bool // Strict returns true if the snap is installed with strict confinement. + UID() int // UID is the user ID to set on config files. + GID() int // GID is the group ID to set on config files. + + StartService(ctx context.Context, serviceName string) error // snapctl start $service + StopService(ctx context.Context, serviceName string) error // snapctl stop $service + RestartService(ctx context.Context, serviceName string) error // snapctl restart $service + + CNIConfDir() string // /etc/cni/net.d + CNIBinDir() string // /opt/cni/bin + CNIPluginsBinary() string // /snap/k8s/current/bin/cni + CNIPlugins() []string // cni plugins built into the cni binary + + KubernetesConfigDir() string // /etc/kubernetes + KubernetesPKIDir() string // /etc/kubernetes/pki + KubeletRootDir() string // /var/lib/kubelet + + ContainerdConfigDir() string // /var/snap/k8s/common/etc/containerd + ContainerdExtraConfigDir() string // /var/snap/k8s/common/etc/containerd/conf.d + ContainerdRegistryConfigDir() string // /var/snap/k8s/common/etc/containerd/hosts.d + ContainerdRootDir() string // /var/snap/k8s/common/var/lib/containerd + ContainerdSocketDir() string // /var/snap/k8s/common/run + ContainerdStateDir() string // /run/containerd + + K8sdStateDir() string // /var/snap/k8s/common/var/lib/k8sd/state + K8sDqliteStateDir() string // /var/snap/k8s/common/var/lib/k8s-dqlite + + ServiceArgumentsDir() string // /var/snap/k8s/common/args + ServiceExtraConfigDir() string // /var/snap/k8s/common/args/conf.d + + Components() map[string]types.Component // available components } diff --git a/src/k8s/pkg/snap/mock/mock.go b/src/k8s/pkg/snap/mock/mock.go index 183ddef11..30cbce7b5 100644 --- a/src/k8s/pkg/snap/mock/mock.go +++ b/src/k8s/pkg/snap/mock/mock.go @@ -2,66 +2,134 @@ package mock import ( "context" - "path/filepath" -) - -// Snap is a generic mock for the snap.Snap interface. -type Snap struct { - Strict bool - StartServiceCalledWith []string - StopServiceCalledWith []string - - WriteServiceArgumentsCalled bool - ServiceArguments map[string]string + "github.com/canonical/k8s/pkg/k8sd/types" + "github.com/canonical/k8s/pkg/snap" +) - PathPrefix string - DataPathPrefix string - CommonPathPrefix string +type Mock struct { + Strict bool + UID int + GID int + KubernetesConfigDir string + KubernetesPKIDir string + KubeletRootDir string + CNIConfDir string + CNIBinDir string + CNIPlugins []string + CNIPluginsBinary string + ContainerdConfigDir string + ContainerdExtraConfigDir string + ContainerdRegistryConfigDir string + ContainerdRootDir string + ContainerdSocketDir string + ContainerdStateDir string + K8sdStateDir string + K8sDqliteStateDir string + ServiceArgumentsDir string + ServiceExtraConfigDir string + Components map[string]types.Component } -func (s *Snap) StartService(_ context.Context, service string) error { - s.StartServiceCalledWith = append(s.StartServiceCalledWith, service) - return nil -} +// Snap is a mock implementation for snap.Snap. +type Snap struct { + StartServiceCalledWith []string + StartServiceErr error + StopServiceCalledWith []string + StopServiceErr error + RestartServiceCalledWith []string + RestartServiceErr error -func (s *Snap) StopService(_ context.Context, service string) error { - s.StopServiceCalledWith = append(s.StopServiceCalledWith, service) - return nil + Mock Mock } -func (s *Snap) RestartService(_ context.Context, service string) error { - return nil +func (s *Snap) StartService(ctx context.Context, name string) error { + if len(s.StartServiceCalledWith) == 0 { + s.StartServiceCalledWith = []string{name} + } else { + s.StartServiceCalledWith = append(s.StartServiceCalledWith, name) + } + return s.StartServiceErr } - -func (s *Snap) ReadServiceArguments(service string) (string, error) { - if s.ServiceArguments == nil { - s.ServiceArguments = make(map[string]string) +func (s *Snap) StopService(ctx context.Context, name string) error { + if len(s.StopServiceCalledWith) == 0 { + s.StopServiceCalledWith = []string{name} + } else { + s.StopServiceCalledWith = append(s.StopServiceCalledWith, name) } - return s.ServiceArguments[service], nil + return s.StopServiceErr } - -func (s *Snap) WriteServiceArguments(service string, b []byte) error { - if s.ServiceArguments == nil { - s.ServiceArguments = make(map[string]string) +func (s *Snap) RestartService(ctx context.Context, name string) error { + if len(s.RestartServiceCalledWith) == 0 { + s.RestartServiceCalledWith = []string{name} + } else { + s.RestartServiceCalledWith = append(s.RestartServiceCalledWith, name) } - s.ServiceArguments[service] = string(b) - s.WriteServiceArgumentsCalled = true - return nil + return s.RestartServiceErr } -func (s *Snap) Path(parts ...string) string { - return filepath.Join(append([]string{s.PathPrefix}, parts...)...) +func (s *Snap) Strict() bool { + return s.Mock.Strict } - -func (s *Snap) DataPath(parts ...string) string { - return filepath.Join(append([]string{s.DataPathPrefix}, parts...)...) +func (s *Snap) UID() int { + return s.Mock.UID } - -func (s *Snap) CommonPath(parts ...string) string { - return filepath.Join(append([]string{s.CommonPathPrefix}, parts...)...) +func (s *Snap) GID() int { + return s.Mock.GID } - -func (s *Snap) IsStrict() bool { - return s.Strict +func (s *Snap) ContainerdConfigDir() string { + return s.Mock.ContainerdConfigDir +} +func (s *Snap) ContainerdRootDir() string { + return s.Mock.ContainerdRootDir +} +func (s *Snap) ContainerdStateDir() string { + return s.Mock.ContainerdStateDir +} +func (s *Snap) ContainerdSocketDir() string { + return s.Mock.ContainerdSocketDir } +func (s *Snap) ContainerdExtraConfigDir() string { + return s.Mock.ContainerdExtraConfigDir +} +func (s *Snap) ContainerdRegistryConfigDir() string { + return s.Mock.ContainerdRegistryConfigDir +} +func (s *Snap) KubernetesConfigDir() string { + return s.Mock.KubernetesConfigDir +} +func (s *Snap) KubernetesPKIDir() string { + return s.Mock.KubernetesPKIDir +} +func (s *Snap) KubeletRootDir() string { + return s.Mock.KubeletRootDir +} +func (s *Snap) CNIConfDir() string { + return s.Mock.CNIConfDir +} +func (s *Snap) CNIBinDir() string { + return s.Mock.CNIBinDir +} +func (s *Snap) CNIPluginsBinary() string { + return s.Mock.CNIPluginsBinary +} +func (s *Snap) CNIPlugins() []string { + return s.Mock.CNIPlugins +} +func (s *Snap) K8sdStateDir() string { + return s.Mock.K8sdStateDir +} +func (s *Snap) K8sDqliteStateDir() string { + return s.Mock.K8sDqliteStateDir +} +func (s *Snap) ServiceArgumentsDir() string { + return s.Mock.ServiceArgumentsDir +} +func (s *Snap) ServiceExtraConfigDir() string { + return s.Mock.ServiceExtraConfigDir +} +func (s *Snap) Components() map[string]types.Component { + return s.Mock.Components +} + +var _ snap.Snap = &Snap{} diff --git a/src/k8s/pkg/snap/services.go b/src/k8s/pkg/snap/services.go deleted file mode 100644 index 63a0fc827..000000000 --- a/src/k8s/pkg/snap/services.go +++ /dev/null @@ -1,163 +0,0 @@ -package snap - -import ( - "context" - "errors" - "fmt" - "os" - "strings" - - "github.com/canonical/k8s/pkg/utils" -) - -var ( - // WorkerServices contains all k8s services that run on a worker node except of k8sd. - WorkerServices = []string{"containerd", "k8s-apiserver-proxy", "kubelet", "kube-proxy"} - // ControlPlaneServices contains all k8s services that run on a control plane except of k8sd. - ControlPlaneServices = []string{ - "containerd", "k8s-dqlite", "kube-apiserver", - "kube-controller-manager", "kube-proxy", - "kube-scheduler", "kubelet", "k8s-apiserver-proxy", - } -) - -// StartWorkerServices starts the worker services. -// StartWorkerServices will return on the first failing service. -func StartWorkerServices(ctx context.Context, snap Snap) error { - for _, service := range WorkerServices { - if err := snap.StartService(ctx, service); err != nil { - return fmt.Errorf("failed to start service %s: %w", service, err) - } - } - return nil -} - -// StartControlPlaneServices starts the control plane services. -// StartControlPlaneServices will return on the first failing service. -func StartControlPlaneServices(ctx context.Context, snap Snap) error { - for _, service := range ControlPlaneServices { - if err := snap.StartService(ctx, service); err != nil { - return fmt.Errorf("failed to start service %s: %w", service, err) - } - } - return nil -} - -// StopWorkerServices stors the worker services. -// StopWorkerServices will return on the first failing service. -func StopWorkerServices(ctx context.Context, snap Snap) error { - for _, service := range WorkerServices { - if err := snap.StopService(ctx, service); err != nil { - return fmt.Errorf("failed to stop service %s: %w", service, err) - } - } - return nil -} - -// StopControlPlaneServices stops the control plane services. -// StopControlPlaneServices will return on the first failing service. -func StopControlPlaneServices(ctx context.Context, snap Snap) error { - for _, service := range ControlPlaneServices { - if err := snap.StopService(ctx, service); err != nil { - return fmt.Errorf("failed to stop service %s: %w", service, err) - } - } - return nil -} - -// GetServiceArgument retrieves the value of a specific argument from the $SNAP_DATA/args/$service file. -// The argument name should include preceding dashes (e.g. "--secure-port"). -// If any errors occur, or the argument is not present, an empty string is returned. -func GetServiceArgument(s Snap, serviceName string, argument string) string { - arguments, err := s.ReadServiceArguments(serviceName) - if err != nil { - return "" - } - - for _, line := range strings.Split(arguments, "\n") { - line = strings.TrimSpace(line) - // ignore empty lines - if line == "" { - continue - } - if key, value := utils.ParseArgumentLine(line); key == argument { - return value - } - } - return "" -} - -// UpdateServiceArguments updates the arguments file for a service. -// UpdateServiceArguments is a no-op if updateList and delete are empty. -// updateList is a map of key-value pairs. It will replace the argument with the new value (or just append). -// delete is a list of arguments to remove completely. The argument is removed if present. -// Returns a boolean whether any of the arguments were changed, as well as any errors that may have occured. -func UpdateServiceArguments(s Snap, serviceName string, updateList []map[string]string, delete []string) (bool, error) { - if updateList == nil { - updateList = []map[string]string{} - } - if delete == nil { - delete = []string{} - } - - // If no updates are requested, exit early - if len(updateList) == 0 && len(delete) == 0 { - return false, nil - } - - deleteMap := make(map[string]struct{}, len(delete)) - for _, k := range delete { - deleteMap[k] = struct{}{} - } - - updateMap := make(map[string]string, len(updateList)) - for _, update := range updateList { - for key, value := range update { - updateMap[key] = value - } - } - - arguments, err := s.ReadServiceArguments(serviceName) - if err != nil && !errors.Is(err, os.ErrNotExist) { - return false, fmt.Errorf("failed to read arguments of service %s: %w", serviceName, err) - } - - changed := false - existingArguments := make(map[string]struct{}, len(arguments)) - newArguments := make([]string, 0, len(arguments)) - for _, line := range strings.Split(arguments, "\n") { - line = strings.TrimSpace(line) - // ignore empty lines - if line == "" { - continue - } - key, oldValue := utils.ParseArgumentLine(line) - existingArguments[key] = struct{}{} - if newValue, ok := updateMap[key]; ok { - // update argument with new value - newArguments = append(newArguments, fmt.Sprintf("%s=%s", key, newValue)) - if oldValue != newValue { - changed = true - } - } else if _, ok := deleteMap[key]; ok { - // remove argument - changed = true - continue - } else { - // no change - newArguments = append(newArguments, line) - } - } - - for key, value := range updateMap { - if _, argExists := existingArguments[key]; !argExists { - changed = true - newArguments = append(newArguments, fmt.Sprintf("%s=%s", key, value)) - } - } - - if err := s.WriteServiceArguments(serviceName, []byte(strings.Join(newArguments, "\n")+"\n")); err != nil { - return false, fmt.Errorf("failed to update arguments for service %s: %q", serviceName, err) - } - return changed, nil -} diff --git a/src/k8s/pkg/snap/snap.go b/src/k8s/pkg/snap/snap.go index e39b3ad03..194049bf9 100644 --- a/src/k8s/pkg/snap/snap.go +++ b/src/k8s/pkg/snap/snap.go @@ -4,9 +4,10 @@ import ( "context" "fmt" "os" - "path/filepath" + "path" "strings" + "github.com/canonical/k8s/pkg/k8sd/types" "github.com/canonical/k8s/pkg/utils" "gopkg.in/yaml.v2" ) @@ -14,59 +15,35 @@ import ( // snap implements the Snap interface. type snap struct { snapDir string - snapDataDir string snapCommonDir string } // NewSnap creates a new interface with the K8s snap. -// NewSnap accepts the $SNAP, $SNAP_DATA and $SNAP_COMMON, directories -func NewSnap(snapDir, snapDataDir, snapCommonDir string, options ...func(s *snap)) Snap { +// NewSnap accepts the $SNAP and $SNAP_COMMON directories +func NewSnap(snapDir, snapCommonDir string) *snap { s := &snap{ snapDir: snapDir, - snapDataDir: snapDataDir, snapCommonDir: snapCommonDir, } - for _, opt := range options { - opt(s) - } return s } -// NewDefaultSnap returns a snap configured with the default snap environment. -func NewDefaultSnap() Snap { - return NewSnap(os.Getenv("SNAP"), os.Getenv("SNAP_DATA"), os.Getenv("SNAP_COMMON")) -} - -type snapContextKey struct{} - -// SnapFromContext extracts the snap instance from the provided context. -// A panic is invoked if there is not snap instance in this context. -func SnapFromContext(ctx context.Context) Snap { - snap, ok := ctx.Value(snapContextKey{}).(Snap) - if !ok { - // This should never happen as the main microcluster state context should contain the snap for k8sd. - // Thus, panic is fine here to avoid cumbersome and unnecessary error checks on client side. - panic("There is no snap value in the given context. Make sure that the context is wrapped with snap.ContextWithSnap.") - } - return snap -} - -// ContextWithSnap adds a snap instance to a given context. -func ContextWithSnap(ctx context.Context, snap Snap) context.Context { - return context.WithValue(ctx, snapContextKey{}, snap) -} - -func (s *snap) Path(parts ...string) string { - return filepath.Join(append([]string{s.snapDir}, parts...)...) +func (s *snap) path(parts ...string) string { + return path.Join(append([]string{s.snapDir}, parts...)...) } -func (s *snap) DataPath(parts ...string) string { - return filepath.Join(append([]string{s.snapDataDir}, parts...)...) +func (s *snap) commonPath(parts ...string) string { + return path.Join(append([]string{s.snapCommonDir}, parts...)...) } -func (s *snap) CommonPath(parts ...string) string { - return filepath.Join(append([]string{s.snapCommonDir}, parts...)...) +// serviceName infers the name of the snapctl daemon from the service name. +// if the serviceName is the snap name `k8s` (=referes to all services) it will return it as is. +func serviceName(serviceName string) string { + if strings.HasPrefix(serviceName, "k8s.") || serviceName == "k8s" { + return serviceName + } + return fmt.Sprintf("k8s.%s", serviceName) } // StartService starts a k8s service. The name can be either prefixed or not. @@ -84,30 +61,13 @@ func (s *snap) RestartService(ctx context.Context, name string) error { return utils.RunCommand(ctx, "snapctl", "restart", serviceName(name)) } -func (s *snap) ReadServiceArguments(serviceName string) (string, error) { - return utils.ReadFile(s.DataPath("args", serviceName)) -} - -func (s *snap) WriteServiceArguments(serviceName string, arguments []byte) error { - return os.WriteFile(s.DataPath("args", serviceName), arguments, 0o660) -} - -// serviceName infers the name of the snapctl daemon from the service name. -// if the serviceName is the snap name `k8s` (=referes to all services) it will return it as is. -func serviceName(serviceName string) string { - if strings.HasPrefix(serviceName, "k8s.") || serviceName == "k8s" { - return serviceName - } - return fmt.Sprintf("k8s.%s", serviceName) -} - type snapcraftYml struct { Confinement string `yaml:"confinement"` } -func (s *snap) IsStrict() bool { +func (s *snap) Strict() bool { var meta snapcraftYml - contents, err := os.ReadFile(s.Path("meta", "snap.yaml")) + contents, err := os.ReadFile(s.path("meta", "snap.yaml")) if err != nil { return false } @@ -116,3 +76,132 @@ func (s *snap) IsStrict() bool { } return meta.Confinement == "strict" } + +func (s *snap) UID() int { + return 0 +} + +func (s *snap) GID() int { + return 0 +} + +func (s *snap) ContainerdConfigDir() string { + return path.Join(s.snapCommonDir, "etc", "containerd") +} + +func (s *snap) ContainerdRootDir() string { + return path.Join(s.snapCommonDir, "var", "lib", "containerd") +} + +func (s *snap) ContainerdSocketDir() string { + return path.Join(s.snapCommonDir, "run") +} + +func (s *snap) ContainerdStateDir() string { + return "/run/containerd" +} + +func (s *snap) CNIConfDir() string { + return "/etc/cni/net.d" +} + +func (s *snap) CNIBinDir() string { + return "/opt/cni/bin" +} + +func (s *snap) CNIPluginsBinary() string { + return path.Join(s.snapDir, "bin", "cni") +} + +func (s *snap) CNIPlugins() []string { + return []string{ + "dhcp", + "host-local", + "static", + "bridge", + "host-device", + "ipvlan", + "loopback", + "macvlan", + "ptp", + "vlan", + "bandwidth", + "firewall", + "portmap", + "sbr", + "tuning", + "vrf", + } +} + +func (s *snap) KubernetesConfigDir() string { + return "/etc/kubernetes" +} + +func (s *snap) KubernetesPKIDir() string { + return "/etc/kubernetes/pki" +} + +func (s *snap) KubeletRootDir() string { + return "/var/lib/kubelet" +} + +func (s *snap) K8sdStateDir() string { + return path.Join(s.snapCommonDir, "var", "lib", "k8sd", "state") +} + +func (s *snap) K8sDqliteStateDir() string { + return path.Join(s.snapCommonDir, "var", "lib", "k8s-dqlite") +} + +func (s *snap) ServiceArgumentsDir() string { + return path.Join(s.snapCommonDir, "args") +} + +func (s *snap) ServiceExtraConfigDir() string { + return path.Join(s.snapCommonDir, "args", "conf.d") +} + +func (s *snap) ContainerdExtraConfigDir() string { + return path.Join(s.snapCommonDir, "etc", "containerd", "conf.d") +} + +func (s *snap) ContainerdRegistryConfigDir() string { + return path.Join(s.snapCommonDir, "etc", "containerd", "hosts.d") +} + +func (s *snap) Components() map[string]types.Component { + return map[string]types.Component{ + "network": { + ReleaseName: "ck-network", + ManifestPath: path.Join(s.snapDir, "k8s", "components", "charts", "cilium-1.14.1.tgz"), + Namespace: "kube-system", + }, + "dns": { + ReleaseName: "ck-dns", + // TODO: fork coredns helm chart so that we can set custom args needed for the rock + ManifestPath: path.Join(s.snapDir, "k8s", "components", "charts", "coredns-1.29.0"), + Namespace: "kube-system", + }, + "storage": { + ReleaseName: "ck-storage", + ManifestPath: path.Join(s.snapDir, "k8s", "components", "charts", "rawfile-csi-0.8.0.tgz"), + Namespace: "kube-system", + }, + "ingress": { + DependsOn: "network", + }, + "gateway": { + ReleaseName: "ck-gateway", + ManifestPath: path.Join(s.snapDir, "k8s", "components", "charts", "gateway-api-0.7.1.tgz"), + Namespace: "kube-system", + }, + "loadbalancer": { + ReleaseName: "ck-loadbalancer", + ManifestPath: path.Join(s.snapDir, "k8s", "components", "charts", "ck-loadbalancer"), + Namespace: "kube-system", + }, + } +} + +var _ Snap = &snap{} diff --git a/src/k8s/pkg/snap/util/arguments.go b/src/k8s/pkg/snap/util/arguments.go new file mode 100644 index 000000000..1003e2eca --- /dev/null +++ b/src/k8s/pkg/snap/util/arguments.go @@ -0,0 +1,110 @@ +package snaputil + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/canonical/k8s/pkg/snap" + "github.com/canonical/k8s/pkg/utils" +) + +func argumentsFileForService(s snap.Snap, serviceName string) string { + return filepath.Join(s.ServiceArgumentsDir(), serviceName) +} + +// GetServiceArgument retrieves the value of a specific argument from the $SNAP_DATA/args/$service file. +// The argument name should include preceding dashes (e.g. "--secure-port"). +// If any errors occur, or the argument is not present, an empty string is returned. +func GetServiceArgument(s snap.Snap, serviceName string, argument string) (string, error) { + arguments, err := os.ReadFile(argumentsFileForService(s, serviceName)) + if err != nil { + return "", fmt.Errorf("failed to read arguments file for service %s: %w", serviceName, err) + } + + for _, line := range strings.Split(string(arguments), "\n") { + line = strings.TrimSpace(line) + // ignore empty lines + if line == "" { + continue + } + if key, value := utils.ParseArgumentLine(line); key == argument { + return value, nil + } + } + return "", nil +} + +// UpdateServiceArguments updates the arguments file for a service. +// UpdateServiceArguments is a no-op if updateList and delete are empty. +// updateList is a map of key-value pairs. It will replace the argument with the new value (or just append). +// delete is a list of arguments to remove completely. The argument is removed if present. +// Returns a boolean whether any of the arguments were changed, as well as any errors that may have occured. +func UpdateServiceArguments(snap snap.Snap, serviceName string, updateMap map[string]string, deleteList []string) (bool, error) { + if updateMap == nil { + updateMap = map[string]string{} + } + if deleteList == nil { + deleteList = []string{} + } + + // If no updates are requested, exit early + if len(updateMap) == 0 && len(deleteList) == 0 { + return false, nil + } + + deleteMap := make(map[string]struct{}, len(deleteList)) + for _, k := range deleteList { + deleteMap[k] = struct{}{} + } + + argumentsFile := argumentsFileForService(snap, serviceName) + arguments, err := os.ReadFile(argumentsFile) + if err != nil && !os.IsNotExist(err) { + return false, fmt.Errorf("failed to read arguments file for service %s: %w", serviceName, err) + } + + changed := false + existingArguments := map[string]struct{}{} + newArguments := []string{} + for _, line := range strings.Split(string(arguments), "\n") { + line = strings.TrimSpace(line) + // ignore empty lines + if line == "" { + continue + } + key, oldValue := utils.ParseArgumentLine(line) + existingArguments[key] = struct{}{} + if newValue, ok := updateMap[key]; ok { + // update argument with new value + newArguments = append(newArguments, fmt.Sprintf("%s=%s", key, newValue)) + if oldValue != newValue { + changed = true + } + } else if _, ok := deleteMap[key]; ok { + // remove argument + changed = true + continue + } else { + // no change + newArguments = append(newArguments, line) + } + } + + for key, value := range updateMap { + if _, argExists := existingArguments[key]; !argExists { + changed = true + newArguments = append(newArguments, fmt.Sprintf("%s=%s", key, value)) + } + } + + // sort arguments so that output is consistent + sort.Strings(newArguments) + + if err := os.WriteFile(argumentsFile, []byte(strings.Join(newArguments, "\n")+"\n"), 0600); err != nil { + return false, fmt.Errorf("failed to write arguments for service %s: %q", serviceName, err) + } + return changed, nil +} diff --git a/src/k8s/pkg/snap/services_test.go b/src/k8s/pkg/snap/util/arguments_test.go similarity index 51% rename from src/k8s/pkg/snap/services_test.go rename to src/k8s/pkg/snap/util/arguments_test.go index 8cd973eba..f2550dd09 100644 --- a/src/k8s/pkg/snap/services_test.go +++ b/src/k8s/pkg/snap/util/arguments_test.go @@ -1,96 +1,106 @@ -package snap_test +package snaputil_test import ( "fmt" "os" + "path/filepath" "testing" - "github.com/canonical/k8s/pkg/snap" "github.com/canonical/k8s/pkg/snap/mock" - + snaputil "github.com/canonical/k8s/pkg/snap/util" . "github.com/onsi/gomega" ) func TestGetServiceArgument(t *testing.T) { - serviceOneArguments := ` + g := NewWithT(t) + dir := t.TempDir() + + s := &mock.Snap{ + Mock: mock.Mock{ + ServiceArgumentsDir: dir, + }, + } + + for svc, args := range map[string]string{ + "service": ` --key=value --key-with-space value2 - --key-with-padding=value3 + --key-with-padding=value3 --multiple=keys --in-the-same-row=this-is-lost -` - serviceTwoArguments := ` + `, + "service2": ` --key=value-of-service-two -` - s := &mock.Snap{ - ServiceArguments: map[string]string{ - "service": serviceOneArguments, - "service2": serviceTwoArguments, - }, - } - if err := os.MkdirAll("testdata/args", 0755); err != nil { - t.Fatal("Failed to setup test directory") +`, + } { + g.Expect(os.WriteFile(filepath.Join(dir, svc), []byte(args), 0600)).To(BeNil()) } + for _, tc := range []struct { - service string - key string - expectedValue string + service string + key string + expectValue string + expectErr bool }{ - {service: "service", key: "--key", expectedValue: "value"}, - {service: "service2", key: "--key", expectedValue: "value-of-service-two"}, - {service: "service", key: "--key-with-padding", expectedValue: "value3"}, - {service: "service", key: "--key-with-space", expectedValue: "value2"}, - {service: "service", key: "--missing", expectedValue: ""}, - {service: "service3", key: "--missing-service", expectedValue: ""}, + {service: "service", key: "--key", expectValue: "value"}, + {service: "service2", key: "--key", expectValue: "value-of-service-two"}, + {service: "service", key: "--key-with-padding", expectValue: "value3"}, + {service: "service", key: "--key-with-space", expectValue: "value2"}, + {service: "service", key: "--missing", expectValue: ""}, + {service: "service3", key: "--missing-service", expectValue: "", expectErr: true}, // NOTE: the final test case documents that arguments in the same row will not be parsed properly. // This is carried over from the original Python code, and probably needs fixing in the future. - {service: "service", key: "--in-the-same-row", expectedValue: ""}, + {service: "service", key: "--in-the-same-row", expectValue: ""}, } { t.Run(fmt.Sprintf("%s/%s", tc.service, tc.key), func(t *testing.T) { g := NewWithT(t) - g.Expect(snap.GetServiceArgument(s, tc.service, tc.key)).To(Equal(tc.expectedValue)) + value, err := snaputil.GetServiceArgument(s, tc.service, tc.key) + if tc.expectErr { + g.Expect(err).ToNot(BeNil()) + } else { + g.Expect(err).To(BeNil()) + g.Expect(value).To(Equal(tc.expectValue)) + } }) } } -type mockSnapFileNotExist struct { - mock.Snap -} - -func (s *mockSnapFileNotExist) ReadServiceArguments(serviceName string) (string, error) { - _, err := os.ReadFile("testdata/fileThatDoesNotExist") - return "", fmt.Errorf("wrapped not found error: %w", err) -} - func TestUpdateServiceArguments(t *testing.T) { t.Run("HandleFileNotExist", func(t *testing.T) { g := NewWithT(t) - s := &mockSnapFileNotExist{ - Snap: mock.Snap{}, + s := &mock.Snap{ + Mock: mock.Mock{ + ServiceArgumentsDir: t.TempDir(), + }, } - changed, err := snap.UpdateServiceArguments(s, "service", []map[string]string{{"--key": "value"}}, nil) + _, err := snaputil.GetServiceArgument(s, "service", "--key") + g.Expect(err).ToNot(BeNil()) + + changed, err := snaputil.UpdateServiceArguments(s, "service", map[string]string{"--key": "value"}, nil) g.Expect(err).To(BeNil()) g.Expect(changed).To(BeTrue()) - g.Expect(s.Snap.ServiceArguments["service"]).To(Equal("--key=value\n")) + value, err := snaputil.GetServiceArgument(s, "service", "--key") + g.Expect(err).To(BeNil()) + g.Expect(value).To(Equal("value")) }) - initialArguments := ` ---key=value ---other=other-value ---with-space value2 -` + initialArguments := map[string]string{ + "--key": "value", + "--other": "other-value", + "--with-space": "value2", + } for _, tc := range []struct { name string - update []map[string]string + update map[string]string delete []string expectedValues map[string]string expectedChange bool }{ { name: "no-change", - update: []map[string]string{{"--key": "value"}}, + update: map[string]string{"--key": "value"}, delete: []string{"--non-existent"}, expectedValues: map[string]string{ "--key": "value", @@ -100,7 +110,7 @@ func TestUpdateServiceArguments(t *testing.T) { }, { name: "no-change-space", - update: []map[string]string{{"--with-space": "value2"}}, + update: map[string]string{"--with-space": "value2"}, delete: []string{}, expectedValues: map[string]string{ "--with-space": "value2", @@ -109,7 +119,7 @@ func TestUpdateServiceArguments(t *testing.T) { }, { name: "simple-update", - update: []map[string]string{{"--key": "new-value"}}, + update: map[string]string{"--key": "new-value"}, delete: []string{}, expectedValues: map[string]string{ "--key": "new-value", @@ -129,7 +139,7 @@ func TestUpdateServiceArguments(t *testing.T) { }, { name: "update-many-delete-one", - update: []map[string]string{{"--key": "new-value"}, {"--other": "other-new-value"}}, + update: map[string]string{"--key": "new-value", "--other": "other-new-value"}, delete: []string{"--with-space"}, expectedValues: map[string]string{ "--key": "new-value", @@ -140,7 +150,7 @@ func TestUpdateServiceArguments(t *testing.T) { }, { name: "update-many-single-list", - update: []map[string]string{{"--key": "new-value", "--other": "other-new-value"}}, + update: map[string]string{"--key": "new-value", "--other": "other-new-value"}, expectedValues: map[string]string{ "--key": "new-value", "--other": "other-new-value", @@ -156,7 +166,7 @@ func TestUpdateServiceArguments(t *testing.T) { }, { name: "new-opt", - update: []map[string]string{{"--new-opt": "opt-value"}}, + update: map[string]string{"--new-opt": "opt-value"}, expectedValues: map[string]string{ "--new-opt": "opt-value", }, @@ -165,24 +175,28 @@ func TestUpdateServiceArguments(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { g := NewWithT(t) + dir := t.TempDir() s := &mock.Snap{ - ServiceArguments: map[string]string{ - "service": initialArguments, + Mock: mock.Mock{ + ServiceArgumentsDir: dir, }, } + changed, err := snaputil.UpdateServiceArguments(s, "service", initialArguments, nil) + g.Expect(err).To(BeNil()) + g.Expect(changed).To(BeTrue()) - changed, err := snap.UpdateServiceArguments(s, "service", tc.update, tc.delete) + changed, err = snaputil.UpdateServiceArguments(s, "service", tc.update, tc.delete) g.Expect(err).To(BeNil()) g.Expect(changed).To(Equal(tc.expectedChange)) for key, expectedValue := range tc.expectedValues { - g.Expect(snap.GetServiceArgument(s, "service", key)).To(Equal(expectedValue)) + g.Expect(snaputil.GetServiceArgument(s, "service", key)).To(Equal(expectedValue)) } t.Run("Reapply", func(t *testing.T) { g := NewWithT(t) - changed, err := snap.UpdateServiceArguments(s, "service", tc.update, tc.delete) + changed, err := snaputil.UpdateServiceArguments(s, "service", tc.update, tc.delete) g.Expect(err).To(BeNil()) g.Expect(changed).To(BeFalse()) }) diff --git a/src/k8s/pkg/snap/util/services.go b/src/k8s/pkg/snap/util/services.go new file mode 100644 index 000000000..78e4a14c0 --- /dev/null +++ b/src/k8s/pkg/snap/util/services.go @@ -0,0 +1,61 @@ +package snaputil + +import ( + "context" + "fmt" + + "github.com/canonical/k8s/pkg/snap" +) + +var ( + // WorkerServices contains all k8s services that run on a worker node except of k8sd. + workerServices = []string{ + "containerd", + "k8s-apiserver-proxy", + "kubelet", + "kube-proxy", + } + // ControlPlaneServices contains all k8s services that run on a control plane except of k8sd. + controlPlaneServices = []string{ + "containerd", + "k8s-dqlite", + "kube-apiserver", + "kube-controller-manager", + "kube-proxy", + "kube-scheduler", + "kubelet", + } +) + +// StartWorkerServices starts the worker services. +// StartWorkerServices will return on the first failing service. +func StartWorkerServices(ctx context.Context, snap snap.Snap) error { + for _, service := range workerServices { + if err := snap.StartService(ctx, service); err != nil { + return fmt.Errorf("failed to start service %s: %w", service, err) + } + } + return nil +} + +// StartControlPlaneServices starts the control plane services. +// StartControlPlaneServices will return on the first failing service. +func StartControlPlaneServices(ctx context.Context, snap snap.Snap) error { + for _, service := range controlPlaneServices { + if err := snap.StartService(ctx, service); err != nil { + return fmt.Errorf("failed to start service %s: %w", service, err) + } + } + return nil +} + +// StopControlPlaneServices stops the control plane services. +// StopControlPlaneServices will return on the first failing service. +func StopControlPlaneServices(ctx context.Context, snap snap.Snap) error { + for _, service := range controlPlaneServices { + if err := snap.StopService(ctx, service); err != nil { + return fmt.Errorf("failed to start service %s: %w", service, err) + } + } + return nil +} diff --git a/src/k8s/pkg/utils/cert/config.go b/src/k8s/pkg/utils/cert/config.go deleted file mode 100644 index 8075e354e..000000000 --- a/src/k8s/pkg/utils/cert/config.go +++ /dev/null @@ -1,8 +0,0 @@ -package cert - -const ( - // The path where kubernetes services certificates will live in. - KubePkiPath = "/etc/kubernetes/pki" - // The path where K8s Dqlite certificates will live in. - K8sDqlitePkiPath = "/var/lib/k8s-dqlite" -) diff --git a/src/k8s/pkg/utils/cert/helpers.go b/src/k8s/pkg/utils/cert/helpers.go deleted file mode 100644 index 73cc7a5d3..000000000 --- a/src/k8s/pkg/utils/cert/helpers.go +++ /dev/null @@ -1,16 +0,0 @@ -package cert - -import ( - "crypto/rand" - "math/big" -) - -// generateSerialNumber returns a random number that can be used for the SerialNumber field in an x509 certificate. -func generateSerialNumber() (*big.Int, error) { - serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) - serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) - if err != nil { - return nil, err - } - return serialNumber, nil -} diff --git a/src/k8s/pkg/utils/cert/manager.go b/src/k8s/pkg/utils/cert/manager.go deleted file mode 100644 index 53898b47b..000000000 --- a/src/k8s/pkg/utils/cert/manager.go +++ /dev/null @@ -1,405 +0,0 @@ -package cert - -import ( - "crypto/rand" - "crypto/rsa" - "crypto/x509/pkix" - "fmt" - "net" - "os" - "path/filepath" - "time" - - "github.com/canonical/k8s/pkg/utils" -) - -// CertificateManager contains and manages certificates that is used by the k8s node. -type CertificateManager struct { - hostname string - defaultIp net.IP - CA *CertKeyPair - FrontProxyCa *CertKeyPair - FrontProxyClient *CertKeyPair - K8sDqlite *CertKeyPair - KubeAdmin *CertKeyPair - KubeApiserver *CertKeyPair - KubeControllerManager *CertKeyPair - KubeProxy *CertKeyPair - KubeScheduler *CertKeyPair - Kubelet *CertKeyPair - KubeletClient *CertKeyPair -} - -// NewCertificateManager returns a new CertificateManager. -func NewCertificateManager() (*CertificateManager, error) { - cm := &CertificateManager{} - - hostname, err := os.Hostname() - if err != nil { - return nil, fmt.Errorf("failed to get hostname: %w", err) - } - cm.hostname = hostname - - defaultIp, err := utils.GetDefaultIP() - if err != nil { - return nil, fmt.Errorf("failed to get default ip: %w", err) - } - cm.defaultIp = defaultIp - - return cm, nil -} - -// GenerateServerCerts generates all the necessary certificates to be used. -func (cm *CertificateManager) GenerateServerCerts() (err error) { - err = cm.generateFrontProxyClient() - if err != nil { - return err - } - cm.generateK8sDqlite() - if err != nil { - return err - } - cm.generateKubeApiserver() - if err != nil { - return err - } - cm.generateKubelet() - if err != nil { - return err - } - cm.generateKubeletClient() - if err != nil { - return err - } - return nil -} - -// GenerateClientCerts generates certificates that can be used to communicate with the apiserver through certificate authentication. -func (cm *CertificateManager) GenerateClientCerts() (err error) { - cm.generateKubeAdmin() - if err != nil { - return err - } - cm.generateKubeControllerManager() - if err != nil { - return err - } - cm.generateKubeProxy() - if err != nil { - return err - } - cm.generateKubeScheduler() - if err != nil { - return err - } - return nil -} - -// GenerateServiceAccountKey generates a private key for the service account. -func (cm *CertificateManager) GenerateServiceAccountKey() error { - key, err := rsa.GenerateKey(rand.Reader, 2048) - if err != nil { - return fmt.Errorf("failed to generate service account key: %w", err) - } - - ckp, err := NewCertKeyPair(nil, key) - if err != nil { - return fmt.Errorf("failed to create cert-key pair: %w", err) - } - - err = ckp.SavePrivateKey(filepath.Join(KubePkiPath, "serviceaccount.key")) - if err != nil { - return fmt.Errorf("failed to save service account key: %w", err) - } - - return nil - -} - -// GenerateCA generates the kubernetes wide certificate authority. -func (cm *CertificateManager) GenerateCA() error { - templ, err := NewCATemplate( - pkix.Name{ - CommonName: "kubernetes-ca", - }, - time.Now().AddDate(10, 0, 0), - 2048, - KubePkiPath, - "ca", - ) - if err != nil { - return fmt.Errorf("failed to create certificate authority template: %w", err) - } - - cert, err := templ.SignAndSave(true, nil) - if err != nil { - return fmt.Errorf("failed to sign and save certificate authority: %w", err) - } - - cm.CA = cert - return nil -} - -// GenerateFrontProxyCA generates the certificate authority for the front-proxy. -func (cm *CertificateManager) GenerateFrontProxyCA() error { - templ, err := NewCATemplate( - pkix.Name{ - CommonName: "kubernetes-front-proxy-ca", - }, - time.Now().AddDate(10, 0, 0), - 2048, - KubePkiPath, - "front-proxy-ca", - ) - if err != nil { - return fmt.Errorf("failed to create front proxy certificate authority template: %w", err) - } - - cert, err := templ.SignAndSave(true, nil) - if err != nil { - return fmt.Errorf("failed to sign and save front proxy certificate authority: %w", err) - } - - cm.FrontProxyCa = cert - return nil -} - -func (cm *CertificateManager) generateFrontProxyClient() error { - templ, err := NewCertificateTemplate( - pkix.Name{ - Organization: []string{"Canonical"}, - OrganizationalUnit: []string{"Canonical"}, - Country: []string{"GB"}, - Province: []string{""}, - Locality: []string{"Canonical"}, - StreetAddress: []string{"Canonical"}, - CommonName: "front-proxy-client", - }, - []string{}, - []net.IP{}, - time.Now().AddDate(10, 0, 0), - 2048, - KubePkiPath, - "front-proxy-client", - ) - if err != nil { - return fmt.Errorf("failed to create front proxy client certificate template: %w", err) - } - - cert, err := templ.SignAndSave(false, cm.FrontProxyCa) - if err != nil { - return fmt.Errorf("failed to sign and save front proxy client certificate: %w", err) - } - - cm.FrontProxyClient = cert - return nil -} - -func (cm *CertificateManager) generateK8sDqlite() error { - templ, err := NewCertificateTemplate( - pkix.Name{ - Organization: []string{"Canonical"}, - OrganizationalUnit: []string{"Canonical"}, - Country: []string{"GB"}, - Province: []string{""}, - Locality: []string{"Canonical"}, - StreetAddress: []string{"Canonical"}, - CommonName: "k8s", - }, - []string{cm.hostname}, - []net.IP{net.IPv4(127, 0, 0, 1)}, - time.Now().AddDate(10, 0, 0), - 2048, - filepath.Join("/var/snap/k8s/common", K8sDqlitePkiPath), // TODO: this must be set from the snap - "cluster", - ) - if err != nil { - return fmt.Errorf("failed to create k8s-dqlite certificate template: %w", err) - } - - cert, err := templ.SignAndSave(true, nil) - if err != nil { - return fmt.Errorf("failed to sign and save k8s-dqlite certificate: %w", err) - } - - cm.K8sDqlite = cert - return nil -} - -func (cm *CertificateManager) generateKubeApiserver() error { - templ, err := NewCertificateTemplate( - pkix.Name{ - CommonName: "kube-apiserver", - }, - []string{"localhost", "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", cm.hostname}, - []net.IP{net.IPv4(127, 0, 0, 1), net.IPv4(10, 152, 183, 1), cm.defaultIp}, - time.Now().AddDate(10, 0, 0), - 2048, - KubePkiPath, - "apiserver", - ) - if err != nil { - return fmt.Errorf("failed to create kube-apiserver certificate template: %w", err) - } - - cert, err := templ.SignAndSave(false, cm.CA) - if err != nil { - return fmt.Errorf("failed to sign and save kube-apiserver certificate: %w", err) - } - - cm.KubeApiserver = cert - return nil -} - -func (cm *CertificateManager) generateKubeAdmin() error { - templ, err := NewCertificateTemplate( - pkix.Name{ - Organization: []string{"system:masters"}, - CommonName: "kubernetes-admin", - }, - []string{}, - []net.IP{}, - time.Now().AddDate(10, 0, 0), - 2048, - KubePkiPath, - "admin", - ) - if err != nil { - return fmt.Errorf("failed to create kube-admin certificate template: %w", err) - } - - cert, err := templ.SignAndSave(false, cm.CA) - if err != nil { - return fmt.Errorf("failed to sign and save kube-admin certificate: %w", err) - } - - cm.KubeAdmin = cert - return nil -} - -func (cm *CertificateManager) generateKubeControllerManager() error { - templ, err := NewCertificateTemplate( - pkix.Name{ - CommonName: "system:kube-controller-manager", - }, - []string{}, - []net.IP{}, - time.Now().AddDate(10, 0, 0), - 2048, - KubePkiPath, - "controller-manager", - ) - if err != nil { - return fmt.Errorf("failed to create kube-controller-manager certificate template: %w", err) - } - - cert, err := templ.SignAndSave(false, cm.CA) - if err != nil { - return fmt.Errorf("failed to sign and save kube-controller-manager certificate: %w", err) - } - - cm.KubeControllerManager = cert - return nil -} - -func (cm *CertificateManager) generateKubeProxy() error { - templ, err := NewCertificateTemplate( - pkix.Name{ - CommonName: "system:kube-proxy", - }, - []string{}, - []net.IP{}, - time.Now().AddDate(10, 0, 0), - 2048, - KubePkiPath, - "proxy", - ) - if err != nil { - return fmt.Errorf("failed to create kube-proxy certificate template: %w", err) - } - - cert, err := templ.SignAndSave(false, cm.CA) - if err != nil { - return fmt.Errorf("failed to sign and save kube-proxy certificate: %w", err) - } - - cm.KubeProxy = cert - return nil -} - -func (cm *CertificateManager) generateKubeScheduler() error { - templ, err := NewCertificateTemplate( - pkix.Name{ - CommonName: "system:kube-scheduler", - }, - []string{}, - []net.IP{}, - time.Now().AddDate(10, 0, 0), - 2048, - KubePkiPath, - "scheduler", - ) - if err != nil { - return fmt.Errorf("failed to create kube-scheduler certificate template: %w", err) - } - - cert, err := templ.SignAndSave(false, cm.CA) - if err != nil { - return fmt.Errorf("failed to sign and save kube-scheduler certificate: %w", err) - } - - cm.KubeScheduler = cert - return nil -} - -func (cm *CertificateManager) generateKubelet() error { - templ, err := NewCertificateTemplate( - pkix.Name{ - Organization: []string{"system:nodes"}, - CommonName: fmt.Sprintf("system:node:%s", cm.hostname), - }, - []string{cm.hostname}, - []net.IP{net.IPv4(127, 0, 0, 1), cm.defaultIp}, - time.Now().AddDate(10, 0, 0), - 2048, - KubePkiPath, - "kubelet", - ) - if err != nil { - return fmt.Errorf("failed to create kubelet certificate template: %w", err) - } - - cert, err := templ.SignAndSave(false, cm.CA) - if err != nil { - return fmt.Errorf("failed to sign and save kubelet certificate: %w", err) - } - - cm.Kubelet = cert - return nil -} - -func (cm *CertificateManager) generateKubeletClient() error { - templ, err := NewCertificateTemplate( - pkix.Name{ - Organization: []string{"system:masters"}, - CommonName: "kube-apiserver-kubelet-client", - }, - []string{}, - []net.IP{}, - time.Now().AddDate(10, 0, 0), - 2048, - KubePkiPath, - "apiserver-kubelet-client", - ) - if err != nil { - return fmt.Errorf("failed to create kubelet client certificate template: %w", err) - } - - cert, err := templ.SignAndSave(false, cm.CA) - if err != nil { - return fmt.Errorf("failed to sign and save kubelet client certificate: %w", err) - } - - cm.KubeletClient = cert - return nil -} diff --git a/src/k8s/pkg/utils/cert/pair.go b/src/k8s/pkg/utils/cert/pair.go deleted file mode 100644 index d696d155a..000000000 --- a/src/k8s/pkg/utils/cert/pair.go +++ /dev/null @@ -1,165 +0,0 @@ -package cert - -import ( - "bytes" - "crypto/rand" - "crypto/rsa" - "crypto/x509" - "encoding/pem" - "errors" - "fmt" - "os" -) - -// CertKeyPair represents a private key and x509 certificate pair. -type CertKeyPair struct { - // Key is the generated RSA private key. - Key *rsa.PrivateKey - // Cert is the x509 certificate. - Cert *x509.Certificate - // KeyPem is the key in PEM format. - KeyPem []byte - // CertPem is the certificate in PEM format. - CertPem []byte -} - -// Sign signs the x509 certificate with the private key. -func (ckp *CertKeyPair) Sign(selfSign bool, ca *CertKeyPair) (err error) { - var derBytes []byte - - if selfSign { - derBytes, err = x509.CreateCertificate(rand.Reader, ckp.Cert, ckp.Cert, &ckp.Key.PublicKey, ckp.Key) - - } else { - derBytes, err = x509.CreateCertificate(rand.Reader, ckp.Cert, ca.Cert, &ckp.Key.PublicKey, ca.Key) - } - if err != nil { - return fmt.Errorf("failed to create x509 certificate: %w", err) - } - - certOut := &bytes.Buffer{} - err = pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) - if err != nil { - return err - } - ckp.CertPem = certOut.Bytes() - - return nil -} - -// NewCertKeyPair returns a new pair from provided certificate and private key. -func NewCertKeyPair(cert *x509.Certificate, privateKey *rsa.PrivateKey) (*CertKeyPair, error) { - ckp := &CertKeyPair{} - ckp.Key = privateKey - ckp.Cert = cert - - privBytes := x509.MarshalPKCS1PrivateKey(privateKey) - - keyOut := &bytes.Buffer{} - err := pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: privBytes}) - if err != nil { - return nil, err - } - ckp.KeyPem = keyOut.Bytes() - - return ckp, nil -} - -// NewCertKeyPairFromPEM returns a new pair from provided PEM-encoded certificate and private key. -func NewCertKeyPairFromPEM(certPEM []byte, keyPem []byte) (*CertKeyPair, error) { - // Decode certificate from PEM - certBlock, _ := pem.Decode(certPEM) - if certBlock == nil { - return nil, errors.New("failed to decode certificate PEM") - } - cert, err := x509.ParseCertificate(certBlock.Bytes) - if err != nil { - return nil, fmt.Errorf("failed to parse certificate: %w", err) - } - - // Decode private key from PEM - keyBlock, _ := pem.Decode(keyPem) - if keyBlock == nil { - return nil, errors.New("failed to decode private key PEM") - } - key, err := x509.ParsePKCS1PrivateKey(keyBlock.Bytes) - if err != nil { - return nil, fmt.Errorf("failed to parse private key: %w", err) - } - - // Create CertKeyPair - ckp := &CertKeyPair{ - Cert: cert, - Key: key, - KeyPem: keyPem, - CertPem: certPEM, - } - - return ckp, nil -} - -// SavePrivateKey marshals the key in PKCS1 format, encodes it as a PEM block and saves it to the given path. -func (ckp *CertKeyPair) SavePrivateKey(path string) error { - keyFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) - if err != nil { - return err - } - defer keyFile.Close() - - _, err = keyFile.Write(ckp.KeyPem) - if err != nil { - return err - } - - return nil -} - -// SaveCertificate encodes the given certificate as a PEM block and saves it to the given path. -func (ckp *CertKeyPair) SaveCertificate(path string) error { - certFile, err := os.Create(path) - if err != nil { - return err - } - defer certFile.Close() - - _, err = certFile.Write(ckp.CertPem) - if err != nil { - return err - } - - return nil -} - -// LoadCertKeyPair loads a key and certificate pair from given paths. -func LoadCertKeyPair(keyPath string, certPath string) (*CertKeyPair, error) { - ckp := &CertKeyPair{} - - dat, err := os.ReadFile(keyPath) - if err != nil { - return nil, err - } - ckp.KeyPem = dat - - pb, _ := pem.Decode(dat) - key, err := x509.ParsePKCS1PrivateKey(pb.Bytes) - if err != nil { - return nil, err - } - ckp.Key = key - - dat, err = os.ReadFile(certPath) - if err != nil { - return nil, err - } - ckp.CertPem = dat - - pb, _ = pem.Decode(dat) - - cert, err := x509.ParseCertificate(pb.Bytes) - if err != nil { - return nil, err - } - ckp.Cert = cert - - return ckp, nil -} diff --git a/src/k8s/pkg/utils/cert/pair_test.go b/src/k8s/pkg/utils/cert/pair_test.go deleted file mode 100644 index 95b58eabb..000000000 --- a/src/k8s/pkg/utils/cert/pair_test.go +++ /dev/null @@ -1,75 +0,0 @@ -package cert - -import ( - "testing" - - . "github.com/onsi/gomega" -) - -func TestNewCertKeyPairFromPEM(t *testing.T) { - certPEM := `-----BEGIN CERTIFICATE----- -MIIDzTCCArWgAwIBAgIQRkxzxTYJLDdSfdeGRE+IeTANBgkqhkiG9w0BAQsFADB2 -MQswCQYDVQQGEwJHQjEJMAcGA1UECBMAMRIwEAYDVQQHEwlDYW5vbmljYWwxEjAQ -BgNVBAkTCUNhbm9uaWNhbDESMBAGA1UEChMJQ2Fub25pY2FsMRIwEAYDVQQLEwlD -YW5vbmljYWwxDDAKBgNVBAMTA2s4czAeFw0yNDAyMDIxMzM0MzFaFw0zNDAyMDIx -MzM0MzFaMHYxCzAJBgNVBAYTAkdCMQkwBwYDVQQIEwAxEjAQBgNVBAcTCUNhbm9u -aWNhbDESMBAGA1UECRMJQ2Fub25pY2FsMRIwEAYDVQQKEwlDYW5vbmljYWwxEjAQ -BgNVBAsTCUNhbm9uaWNhbDEMMAoGA1UEAxMDazhzMIIBIjANBgkqhkiG9w0BAQEF -AAOCAQ8AMIIBCgKCAQEA5yD+C2smIuZLoEYAyKzNpQ9ZQYAS3GbIV0Z/+sqplxgR -sv1ZXUhaO5SQPtaZL+1CMOno/OBnBP3GO/3aO746hdWPgNzQoNAOrOh3HUr7x5qV -SZbExd6igqNq9bGbn6F4zRdaxOeojcRxgaGAQVMm61nSy4owBtRFVmClOnjRKzpS -D5qR272i/7pNj38VPOfGSg5K1PHElE3OL/Dlw7DbQ0tkbMaIXip0i3+4xGUmyffW -OZJHbKLi+P+OGT+4OLKVB1GCWap41vhXWtGWvg7lqG5gru00iio5sgaXNs7tZ4dV -+TgxXCQ2ZhPLV3U6MiHqQiWf9MpBxmXnXfoNyLsnTwIDAQABo1cwVTAOBgNVHQ8B -Af8EBAMCBLAwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMAwGA1UdEwEB -/wQCMAAwFgYDVR0RBA8wDYIFYmFzaWOHBH8AAAEwDQYJKoZIhvcNAQELBQADggEB -AKo7mE0T+4jS7RyphiMJLnEybf3dcEuSWP6nU39J8sGsCwR3Q0xCIimacI3jCJ6v -kkW4/hGtAlKgP0Ti5Q5XTm6MvthYzDCNPjalaS/H8l00mTPIgECan3XB6us+Bbwx -yaWoaaawsUannQhBfiN+xKy5CxqZ6zIJsJbsoxSC+K6se1LlkahcFIAUpcCAx5bG -KPhdcVmZ0GMdsOBBrdxCsto0ywPtEcug1zSml0+UTp7jSMHYeECERCuKKN1HJ5X6 -PU2AtoaNVDwOsThQbRZ9c4iEUySAyK0300F11/8eZmc52uvTVdqHj2e2LvUNe3fU -pSy2WGTNK1Ybmhp0uvEdNxc= ------END CERTIFICATE-----` - - keyPEM := ` ------BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEA4IuXnCSBF4ce8HIILaQkygxbqchyEzlGQTNUGmHXJpz+kll+ -7GxhiYaxj2GgwXJMJhwnfU7Gum+r12gMJn/J6tC48qDAoL0XljmrbNh9PTIIBUMJ -iw0FH5l7D0Webubd0fG5rKNGN13ZH5y6XsLUTRWweod4rS2hzHPZpprXMaTDL+RZ -w1vqeVqtcPU6L6TBGEEL7XztamRb9U48vt8aVJeqe2LIgdxa5LGDkHTNbZutgLfr -CgkRwYmGEsE1/CuaP0EVuKIoEaq4m/ZTIO0gSBPKN5HrV8TXarwSToz1uXywzyEw -1Y66bOW9N8x12JVv/DaY9R8h9+HCzZEzdVwJpwIDAQABAoIBADrq5cEGowi1X5zf -jt3K8AxD6dlGywfYQ2LgQPUbLwTmtLVZ3j/SwFUBjp18lF2ty+VEBeiPPv20R3ah -ym5foW+HjL+9Bk4mz4WPZDePJm1kL06SJHLj27BK6Us8jTG0SgfIUVvHC6mDanEt -AonSvTnssv2zhiqKiqXG0BkzD7vqbWitjhGKB0rp+tFSmWBtB+1u8owX63ApDYda -rTLbOXV2IxPqYtfRCMWsOfdd9VK7qjfkrhTRYLeqljwXjTQNr7ky9uqunaEmNFKk -3aM2Ov7rWr2ICd6rJEiA+h2YJhbB8a8gA8nJmx6dNP6iqw/XjtiZlGL70oYPf7yM -DpGwo0ECgYEA/50zbobAUoDLBQ2WJk5uQvUEQOYcM5I8IcDHUNePRkM8fVlvOHem -MySWqvdr9tRM3VZwlNKk87x+1Crw5NBGANA8GLAHx4t1Nwj3gZw1SazlH6wFHADR -XcUYqE/Sali3fDaW+ccZXx6UhSfRRAPzUE9/hfCMhCO2d/M4DQTaDokCgYEA4OJh -+db054OWqKMCBbi+UXB9KmFwNzF5EcP6FJyw41ySfTHyv1kszB/I6ZNCaWBOFZsF -U8/nR79wjfUvhCasURk6sxvu/byFGdPixEwmFvYUBfBxmuV+5PWIsnyzOYPI1NNZ -SD0+M3bstOEAlpv2pJXIVCfhyzyHJjrR/Nk0yq8CgYEAj77gqHxA9WSWRx4v3RTo -LuFI5hJBs8K97CFUNSMz4Eh5YOiFglTO8x3VWQnQ/jq/iw1MPHUE7EiJbllDpPl3 -FIgF88Ayb8X+QdfaEFo/IVKIezoqmWfgVYI8bpKM+t8vODwexRJxMyuoTAVrSnuK -PhSm2zS+YKUQvP3a/H9I/TkCgYEAyDA+T43PDZjSMOSLFFTU9uJQSb3biwZ7ZBk1 -Mcwamwr5TIF+OmBDxKI209bHM88LM6iAIY/drrz4kSZGWjmjA76VxoODSFTdl5RL -Nbsj3STJxk+4kc7iGyeHvHvNf9GUogBSZkA+csnXBV2WjHviH7lGT8QA+E61cI7E -B3XDzPkCgYB7UxW408teQhcnKl0/hu1H51AUHFXVsnlBudiQ1k/QW6GOGtOIGna5 -dh+UY17lutiawCwXd09oT5+S1m/xyP9Z82l4NgcIh6W2V5d2OBqiWD3YcbdCTsBH -G8AemffxTnBGRlQGY01cveIfpOZ5VCqefDb4tkahGmpsr4LjsssaLw== ------END RSA PRIVATE KEY-----` - - g := NewWithT(t) - - // Create CertKeyPair using NewCertKeyPairFromPEM function - ckp, err := NewCertKeyPairFromPEM([]byte(certPEM), []byte(keyPEM)) - if err != nil { - t.Fatal("Failed to create CertKeyPair from PEM:", err) - } - - g.Expect(ckp.Cert.Subject.CommonName).To(Equal("k8s")) - g.Expect(ckp.Cert.Issuer.Organization).To(Equal([]string{"Canonical"})) - g.Expect(ckp.Cert.Issuer.Locality).To(Equal([]string{"Canonical"})) - g.Expect(ckp.Cert.Issuer.Country).To(Equal([]string{"GB"})) -} diff --git a/src/k8s/pkg/utils/cert/template.go b/src/k8s/pkg/utils/cert/template.go deleted file mode 100644 index 80affc1ba..000000000 --- a/src/k8s/pkg/utils/cert/template.go +++ /dev/null @@ -1,121 +0,0 @@ -package cert - -import ( - "crypto/rand" - "crypto/rsa" - "crypto/x509" - "crypto/x509/pkix" - "fmt" - "net" - "os" - "path/filepath" - "time" -) - -// CertificateTemplate represents a certificate that will be generated. -type CertificateTemplate struct { - // The x509 certificate to sign. - cert *x509.Certificate - // The size of RSA key that will be generated. - pkiBits int - // The path where the generated private key and certificate will reside. - certFolder string - // The name of the .key and .crt files. - certName string -} - -// NewCertificateTemplate returns a certificate template that can be signed later on. -// For input it takes: -// - subject (the subject, e.g. containing CN field for the certificate) -// - dnsNames (used in subjectAltName) -// - ipAddresses (used in subjectAltName) -// - expiry (the expiry date of the certificate, e.g. for a certificate valid for 10 years would be time.Now().AddDate(10, 0, 0)) -// - pkiBits (the bit size of the RSA private key) -// - certFolder (the folder where the generated private key and certificate will reside) -// - certName (the name that will be used to create the .key and .crt files) -func NewCertificateTemplate(subject pkix.Name, dnsNames []string, ipAddresses []net.IP, expiry time.Time, pkiBits int, certFolder, certName string) (*CertificateTemplate, error) { - serialNumber, err := generateSerialNumber() - if err != nil { - return nil, fmt.Errorf("failed to generate serial number for certificate template: %w", err) - } - - return &CertificateTemplate{ - cert: &x509.Certificate{ - SerialNumber: serialNumber, - Subject: subject, - NotBefore: time.Now(), - NotAfter: expiry, - IsCA: false, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, - KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDataEncipherment | x509.KeyUsageDigitalSignature, - BasicConstraintsValid: true, - DNSNames: dnsNames, - IPAddresses: ipAddresses, - }, - pkiBits: pkiBits, - certFolder: certFolder, - certName: certName, - }, nil -} - -// NewCaTemplate returns a certificate template that will create a certificate authority. -// For input it takes: -// - subject -// - expiry (the expiry date of the certificate, e.g. for a certificate valid for 10 years would be time.Now().AddDate(10, 0, 0)) -// - pkiBits (the bit size of the RSA private key) -// - certFolder (the folder where the generated private key and certificate will reside) -// - certName (the name that will be used to create the .key and .crt files) -func NewCATemplate(subject pkix.Name, expiry time.Time, pkiBits int, certFolder, certName string) (*CertificateTemplate, error) { - serialNumber, err := generateSerialNumber() - if err != nil { - return nil, fmt.Errorf("failed to generate serial number for certificate template: %w", err) - } - - return &CertificateTemplate{ - cert: &x509.Certificate{ - SerialNumber: serialNumber, - Subject: subject, - NotBefore: time.Now(), - NotAfter: expiry, - IsCA: true, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, - KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, - BasicConstraintsValid: true, - }, - pkiBits: pkiBits, - certFolder: certFolder, - certName: certName, - }, nil -} - -// SignAndSaveCertificate creates, signs and saves a certificate from the template. -func (ctpl *CertificateTemplate) SignAndSave(selfSign bool, ca *CertKeyPair) (*CertKeyPair, error) { - key, err := rsa.GenerateKey(rand.Reader, ctpl.pkiBits) - if err != nil { - return nil, fmt.Errorf("failed to create RSA private key: %w", err) - } - - ckp, err := NewCertKeyPair(ctpl.cert, key) - if err != nil { - return nil, fmt.Errorf("failed to create cert-key pair: %w", err) - } - - err = ckp.Sign(selfSign, ca) - if err != nil { - return nil, fmt.Errorf("failed to sign certificate: %w", err) - } - - if err := os.MkdirAll(ctpl.certFolder, 0700); err != nil { - return nil, fmt.Errorf("failed to create dir: %w", err) - } - - if err := ckp.SavePrivateKey(filepath.Join(ctpl.certFolder, fmt.Sprintf("%s.key", ctpl.certName))); err != nil { - return nil, fmt.Errorf("failed to save private key: %w", err) - } - - if err := ckp.SaveCertificate(filepath.Join(ctpl.certFolder, fmt.Sprintf("%s.crt", ctpl.certName))); err != nil { - return nil, fmt.Errorf("failed to save certificate: %w", err) - } - - return ckp, nil -} diff --git a/src/k8s/pkg/utils/dqlite/k8s-dqlite.go b/src/k8s/pkg/utils/dqlite/k8s-dqlite.go index 788f02b7f..f7f63fe6a 100644 --- a/src/k8s/pkg/utils/dqlite/k8s-dqlite.go +++ b/src/k8s/pkg/utils/dqlite/k8s-dqlite.go @@ -3,6 +3,7 @@ package dqlite import ( "context" "fmt" + "path" "github.com/canonical/go-dqlite/client" "github.com/canonical/k8s/pkg/snap" @@ -14,7 +15,7 @@ import ( // This should be done by using the go-dqlite client implementation. // However, when I tried to use it the client connects, but returns an empty cluster member list. func GetK8sDqliteClusterMembers(ctx context.Context, snap snap.Snap) ([]NodeInfo, error) { - c, err := client.DefaultNodeStore(snap.CommonPath("/var/lib/k8s-dqlite/cluster.yaml")) + c, err := client.DefaultNodeStore(path.Join(snap.K8sDqliteStateDir(), "cluster.yaml")) if err != nil { return nil, fmt.Errorf("failed to get k8s-dqlite datastore: %w", err) } diff --git a/src/k8s/pkg/utils/file.go b/src/k8s/pkg/utils/file.go index 81ee38d5a..700a924b2 100644 --- a/src/k8s/pkg/utils/file.go +++ b/src/k8s/pkg/utils/file.go @@ -3,45 +3,14 @@ package utils import ( "bufio" "fmt" - "html/template" "io" - "io/fs" "os" - "path/filepath" "sort" "strings" - "syscall" "github.com/moby/sys/mountinfo" ) -// TemplateAndSave compiles a template with the data and saves it to the given target path. -func TemplateAndSave(tmplFile string, data any, target string) error { - tmpl := template.Must(template.ParseFiles(tmplFile)) - - f, err := os.Create(target) - if err != nil { - return err - } - - return tmpl.Execute(f, data) -} - -// FileExists returns true if the specified path exists. -func FileExists(path string) bool { - _, err := os.Stat(path) - return !os.IsNotExist(err) -} - -// ReadFile returns the file contents as a string. -func ReadFile(path string) (string, error) { - b, err := os.ReadFile(path) - if err != nil { - return "", fmt.Errorf("failed to read %s: %w", path, err) - } - return string(b), nil -} - // ParseArgumentLine parses a command-line argument from a single line. // The returned key includes any dash prefixes. func ParseArgumentLine(line string) (key string, value string) { @@ -112,86 +81,6 @@ func SerializeArgumentFile(arguments map[string]string, path string) error { return nil } -// ChmodRecursive changes permissions of files and folders recursively. -func ChmodRecursive(name string, mode fs.FileMode) error { - err := filepath.WalkDir(name, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return fmt.Errorf("failed to walk into path: %w", err) - } - - err = os.Chmod(path, mode) - if err != nil { - return fmt.Errorf("failed to change permissions: %w", err) - } - - return nil - }) - if err != nil { - return fmt.Errorf("failed to change permissions recursively: %w", err) - } - - return nil -} - -// CopyDirectory recursively copies files and directories from the given srcDir. -// -// https://stackoverflow.com/a/56314145 -func CopyDirectory(scrDir, dest string) error { - entries, err := os.ReadDir(scrDir) - if err != nil { - return err - } - for _, entry := range entries { - sourcePath := filepath.Join(scrDir, entry.Name()) - destPath := filepath.Join(dest, entry.Name()) - - fileInfo, err := os.Stat(sourcePath) - if err != nil { - return err - } - - stat, ok := fileInfo.Sys().(*syscall.Stat_t) - if !ok { - return fmt.Errorf("failed to get raw syscall.Stat_t data for '%s'", sourcePath) - } - - switch fileInfo.Mode() & os.ModeType { - case os.ModeDir: - if err := CreateIfNotExists(destPath, 0o755); err != nil { - return err - } - if err := CopyDirectory(sourcePath, destPath); err != nil { - return err - } - case os.ModeSymlink: - if err := CopySymLink(sourcePath, destPath); err != nil { - return err - } - default: - if err := CopyFile(sourcePath, destPath); err != nil { - return err - } - } - - if err := os.Lchown(destPath, int(stat.Uid), int(stat.Gid)); err != nil { - return err - } - - fInfo, err := entry.Info() - if err != nil { - return err - } - - isSymlink := fInfo.Mode()&os.ModeSymlink != 0 - if !isSymlink { - if err := os.Chmod(destPath, fInfo.Mode()); err != nil { - return err - } - } - } - return nil -} - func CopyFile(srcFile, dstFile string) error { out, err := os.Create(dstFile) if err != nil { @@ -215,34 +104,6 @@ func CopyFile(srcFile, dstFile string) error { return nil } -func Exists(filePath string) bool { - if _, err := os.Stat(filePath); os.IsNotExist(err) { - return false - } - - return true -} - -func CreateIfNotExists(dir string, perm os.FileMode) error { - if Exists(dir) { - return nil - } - - if err := os.MkdirAll(dir, perm); err != nil { - return fmt.Errorf("failed to create directory: '%s', error: '%s'", dir, err.Error()) - } - - return nil -} - -func CopySymLink(source, dest string) error { - link, err := os.Readlink(source) - if err != nil { - return fmt.Errorf("could not read symlink: %w", err) - } - return os.Symlink(link, dest) -} - // GetMountPath returns the first mountpath for a given filesystem type. func GetMountPath(fsType string) (string, error) { mounts, err := mountinfo.GetMounts(mountinfo.FSTypeFilter(fsType)) diff --git a/src/k8s/pkg/utils/file_test.go b/src/k8s/pkg/utils/file_test.go index 3dbc8cd4a..afad9f264 100644 --- a/src/k8s/pkg/utils/file_test.go +++ b/src/k8s/pkg/utils/file_test.go @@ -1,7 +1,6 @@ package utils_test import ( - "fmt" "os" "path/filepath" "testing" @@ -10,42 +9,6 @@ import ( . "github.com/onsi/gomega" ) -func TestFileExists(t *testing.T) { - testFilePath := fmt.Sprintf("%s/myfile", t.TempDir()) - _, err := os.Create(testFilePath) - if err != nil { - t.Fatal("Failed to create test file") - } - - if !utils.FileExists(testFilePath) { - t.Fatal("File should exist but it does not") - } - - if err := os.Remove(testFilePath); err != nil { - t.Fatalf("Failed to delete test file: %s", err) - } - - if utils.FileExists(testFilePath) { - t.Fatal("File should not exist but it does") - } -} - -func TestReadFile(t *testing.T) { - testFilePath := fmt.Sprintf("%s/test-read-file", t.TempDir()) - if err := os.WriteFile(testFilePath, []byte(`my text`), 0644); err != nil { - t.Fatal("Failed to create test file") - } - - contents, err := utils.ReadFile(testFilePath) - if err != nil { - t.Fatalf("Failed to read test file: %s", err) - } - if contents != "my text" { - t.Fatalf("Test file should contain 'my test' but it contained '%s'", contents) - } - os.Remove(testFilePath) -} - func TestParseArgumentLine(t *testing.T) { for _, tc := range []struct { line, key, value string diff --git a/src/k8s/pkg/utils/k8s/client.go b/src/k8s/pkg/utils/k8s/client.go index bdd3300e8..a711a3717 100644 --- a/src/k8s/pkg/utils/k8s/client.go +++ b/src/k8s/pkg/utils/k8s/client.go @@ -2,7 +2,9 @@ package k8s import ( "fmt" + "path" + "github.com/canonical/k8s/pkg/snap" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" ) @@ -18,8 +20,8 @@ type k8sClient struct { // There is no way for the user to overwrite this kubeconfig. // We might need to add this functionality similar to `k8s kubectl`. // However, simply querying the KUBECONFIG env will not work for remote clients. -func NewClient() (*k8sClient, error) { - config, err := clientcmd.BuildConfigFromFlags("", "/etc/kubernetes/admin.conf") +func NewClient(snap snap.Snap) (*k8sClient, error) { + config, err := clientcmd.BuildConfigFromFlags("", path.Join(snap.KubernetesConfigDir(), "admin.conf")) if err != nil { return nil, fmt.Errorf("failed to create k8s kubeconfig: %w", err) } diff --git a/src/k8s/pkg/utils/net.go b/src/k8s/pkg/utils/net.go deleted file mode 100644 index 9084c859b..000000000 --- a/src/k8s/pkg/utils/net.go +++ /dev/null @@ -1,18 +0,0 @@ -package utils - -import ( - "fmt" - "net" - - "github.com/canonical/lxd/lxd/util" -) - -// GetDefaultIP returns the IP address of the default interface. -func GetDefaultIP() (net.IP, error) { - parsed := net.ParseIP(util.NetworkInterfaceAddress()) - if parsed == nil { - return nil, fmt.Errorf("failed to get the IP address of the default interface") - } - - return parsed, nil -} diff --git a/src/k8s/pkg/utils/sanitise.go b/src/k8s/pkg/utils/sanitise.go deleted file mode 100644 index d330440cf..000000000 --- a/src/k8s/pkg/utils/sanitise.go +++ /dev/null @@ -1,20 +0,0 @@ -package utils - -import "fmt" - -// sanitiseMap converts a map with interface{} keys to a map with string keys. -// This is useful for preparing data for use with the Helm client, which requires -// map keys to be strings. Nested maps are also recursively processed to ensure -// all keys are converted to strings. -func SanitiseMap(m map[interface{}]interface{}) map[string]interface{} { - result := map[string]interface{}{} - for key, value := range m { - switch t := value.(type) { - case map[interface{}]interface{}: - result[fmt.Sprint(key)] = SanitiseMap(t) - default: - result[fmt.Sprint(key)] = value - } - } - return result -} diff --git a/tests/e2e/lxd-profile.yaml b/tests/e2e/lxd-profile.yaml index 40904fd0a..0cc3c658c 100644 --- a/tests/e2e/lxd-profile.yaml +++ b/tests/e2e/lxd-profile.yaml @@ -18,10 +18,6 @@ devices: path: /dev/kmsg source: /dev/kmsg type: unix-char - aadisable3: - path: /sys/fs/bpf - source: /sys/fs/bpf - type: disk aadisable4: path: /proc/sys/net/netfilter/nf_conntrack_max source: /proc/sys/net/netfilter/nf_conntrack_max