From fe5a47100cd3f71f1ef53990a25cac705a02abec Mon Sep 17 00:00:00 2001 From: Pedro Coutinho Date: Wed, 7 Jun 2023 15:56:15 -0700 Subject: [PATCH 01/16] [WIP] Add GA support for Windows hostprocess Build on the changes from https://github.com/projectcalico/calico/pull/7260: - Add a Windows cni-plugin hostprocess image, which will install the cni binaries and config on the host. - Add a Windows node hostprocess image, which no longer needs to match the hosts' Windows version (e.g. only 1809 needs to be used and is supported on 2022). --- cni-plugin/Dockerfile-windows | 35 ++++ cni-plugin/Makefile | 76 ++++++-- cni-plugin/cmd/calico/calico.go | 2 +- cni-plugin/pkg/install/install.go | 96 ++++++----- cni-plugin/pkg/install/install_linux.go | 48 ++++++ cni-plugin/pkg/install/install_windows.go | 163 ++++++++++++++++++ lib.Makefile | 35 ++++ libcalico-go/lib/winutils/winutils.go | 41 +++++ metadata.mk | 5 +- node/.dockerignore | 6 + node/Dockerfile-windows | 44 +++++ node/Makefile | 70 ++++---- node/pkg/cni/token_watch.go | 7 +- node/pkg/winupgrade/upgrade.go | 33 +--- .../CalicoWindows/confd/confd-service.ps1 | 1 - .../CalicoWindows/config-hpc.ps1 | 61 +++++++ 16 files changed, 599 insertions(+), 124 deletions(-) create mode 100644 cni-plugin/Dockerfile-windows create mode 100644 cni-plugin/pkg/install/install_linux.go create mode 100644 cni-plugin/pkg/install/install_windows.go create mode 100644 libcalico-go/lib/winutils/winutils.go create mode 100644 node/Dockerfile-windows create mode 100644 node/windows-packaging/CalicoWindows/config-hpc.ps1 diff --git a/cni-plugin/Dockerfile-windows b/cni-plugin/Dockerfile-windows new file mode 100644 index 00000000000..b2188b3afa6 --- /dev/null +++ b/cni-plugin/Dockerfile-windows @@ -0,0 +1,35 @@ +# Copyright (c) 2023 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG WINDOWS_VERSION +FROM mcr.microsoft.com/windows/nanoserver:${WINDOWS_VERSION} + +ARG GIT_VERSION=unknown + +LABEL name="Calico Networking for CNI" \ + vendor="Project Calico" \ + version=$GIT_VERSION \ + release="1" \ + summary="Calico Networking for CNI" \ + description="Calico Networking for CNI includes a CNI networking plugin and CNI IPAM plugin" \ + maintainer="maintainers@projectcalico.org" + +ADD licenses/ /licenses +ADD LICENSE /licenses/ + +ADD bin/windows/ /opt/cni/bin/ + +ENV PATH=$PATH:/opt/cni/bin +WORKDIR /opt/cni/bin +CMD ["/opt/cni/bin/install.exe"] diff --git a/cni-plugin/Makefile b/cni-plugin/Makefile index e8a4c03f5b8..8ed1ee66afb 100644 --- a/cni-plugin/Makefile +++ b/cni-plugin/Makefile @@ -8,6 +8,7 @@ LOCAL_CHECKS=check-boring-ssl # Name of the images. # e.g., /: CNI_PLUGIN_IMAGE ?=cni +CNI_PLUGIN_IMAGE_WINDOWS ?=cni-windows BUILD_IMAGES ?=$(CNI_PLUGIN_IMAGE) ############################################################################### @@ -38,6 +39,7 @@ CNI_SPEC_VERSION?=0.3.1 # Markers for various images we produce. DEPLOY_CONTAINER_STATIC_MARKER=cni_deploy_container-$(ARCH).created DEPLOY_CONTAINER_FIPS_MARKER=cni_deploy_container-$(ARCH)-fips.created +DEPLOY_CONTAINER_MARKER_WINDOWS=cni_deploy_container-windows.created # Set FIPS to true to enable a FIPS compliant build using BoringSSL and CGO. # Note that this produces binaries that are dynamically linked, and so have dependencies @@ -55,23 +57,31 @@ BIN=bin/$(ARCH) DEPLOY_CONTAINER_MARKER=$(DEPLOY_CONTAINER_STATIC_MARKER) endif -.PHONY: clean -clean: +# The directory for windows image tarball +WINDOWS_DIST = dist/windows + +.PHONY: clean clean-windows +clean: clean-windows # Clean .created files which indicate images / releases have been built. find . -name '.*.created*' -type f -delete find . -name '.*.published*' -type f -delete - rm -rf $(BIN) bin $(DEPLOY_CONTAINER_MARKER) .go-pkg-cache pkg/install/install.test + rm -rf $(BIN) bin $(DEPLOY_CONTAINER_MARKER) .go-pkg-cache pkg/install/install.test rm -f *.created rm -rf config/ + rm -rf dist/ + +clean-windows: clean-windows-builder + rm -rf $(WINDOWS_BIN) $(WINDOWS_DIST) $(DEPLOY_CONTAINER_MARKER_WINDOWS) ############################################################################### # Building the binary ############################################################################### +.PHONY: build build: $(BIN)/install $(BIN)/calico $(BIN)/calico-ipam ifeq ($(ARCH),amd64) # Go only supports amd64 for Windows builds. -BIN_WIN=bin/windows -build: $(BIN_WIN)/calico.exe $(BIN_WIN)/calico-ipam.exe +WINDOWS_BIN=bin/windows +build: $(WINDOWS_BIN)/install.exe $(WINDOWS_BIN)/calico.exe $(WINDOWS_BIN)/calico-ipam.exe endif # If ARCH is arm based, find the requested version/variant ifeq ($(word 1,$(subst v, ,$(ARCH))),arm) @@ -97,20 +107,24 @@ else endif ## Build the Calico network plugin and ipam plugins for Windows -$(BIN_WIN)/calico.exe $(BIN_WIN)/calico-ipam.exe: $(SRC_FILES) +$(WINDOWS_BIN)/install.exe: $(SRC_FILES) + -mkdir -p $(WINDOWS_BIN) $(DOCKER_RUN) \ -e GOOS=windows \ + -e GOARCH=amd64 \ -e CGO_ENABLED=1 \ - $(CALICO_BUILD) sh -c '$(GIT_CONFIG_SSH) \ - go build -v -buildvcs=false -o $(BIN_WIN)/calico.exe -ldflags "$(LDFLAGS)" $(PACKAGE_NAME)/cmd/calico && \ - go build -v -buildvcs=false -o $(BIN_WIN)/calico-ipam.exe -ldflags "$(LDFLAGS)" $(PACKAGE_NAME)/cmd/calico' + $(CALICO_BUILD) sh -c '$(GIT_CONFIG_SSH) \ + go build -v -buildvcs=false -o $(WINDOWS_BIN)/install.exe -ldflags "$(LDFLAGS)" $(PACKAGE_NAME)/cmd/calico' +$(WINDOWS_BIN)/calico-ipam.exe $(WINDOWS_BIN)/calico.exe: $(WINDOWS_BIN)/install.exe + cp "$<" "$@" ############################################################################### # Building the image ############################################################################### image: $(DEPLOY_CONTAINER_MARKER) -image-all: $(addprefix sub-image-,$(VALIDARCHES)) sub-image-fips-amd64 +image-tar-windows: $(DEPLOY_CONTAINER_MARKER_WINDOWS) +image-all: $(addprefix sub-image-,$(VALIDARCHES)) sub-image-fips-amd64 image-tar-windows sub-image-%: $(MAKE) image ARCH=$* sub-image-fips-%: @@ -128,17 +142,35 @@ $(DEPLOY_CONTAINER_FIPS_MARKER): Dockerfile.$(ARCH) build fetch-cni-bins $(MAKE) retag-build-images-with-registries VALIDARCHES=$(ARCH) IMAGETAG=latest-fips LATEST_IMAGE_TAG=latest-fips touch $@ -.PHONY: fetch-cni-bins +$(DEPLOY_CONTAINER_MARKER_WINDOWS): setup-windows-builder Dockerfile-windows build fetch-win-cni-bins + -mkdir -p $(WINDOWS_DIST) + docker buildx build \ + --platform windows/amd64 \ + --output=type=docker,dest=$(WINDOWS_DIST)/calico-cni-plugin-windows-$(GIT_VERSION).tar \ + -t $(CNI_PLUGIN_IMAGE_WINDOWS):latest \ + --pull \ + --no-cache \ + --build-arg GIT_VERSION=$(GIT_VERSION) \ + --build-arg WINDOWS_VERSION=$(WINDOWS_VERSION) \ + -f Dockerfile-windows . + +.PHONY: fetch-cni-bins fetch-win-cni-bins fetch-cni-bins: $(BIN)/flannel $(BIN)/loopback $(BIN)/host-local $(BIN)/portmap $(BIN)/tuning $(BIN)/bandwidth +fetch-win-cni-bins: $(WINDOWS_BIN)/flannel.exe $(BIN)/loopback $(BIN)/host-local $(BIN)/portmap $(BIN)/tuning $(BIN)/bandwidth: - mkdir -p $(BIN) + -mkdir -p $(BIN) $(CURL) -L --retry 5 $(CNI_ARTIFACTS_URL)/$(CNI_VERSION)/cni-plugins-linux-$(subst v7,,$(ARCH))-$(CNI_VERSION).tgz | tar -xz -C $(BIN) ./loopback ./host-local ./portmap ./tuning ./bandwidth $(BIN)/flannel: - mkdir -p $(BIN) + -mkdir -p $(BIN) $(CURL) -L --retry 5 https://github.com/flannel-io/cni-plugin/releases/download/$(FLANNEL_VERSION)/cni-plugin-flannel-linux-$(subst v7,,$(ARCH))-$(FLANNEL_VERSION).tgz | tar -xz -C $(BIN) ./flannel-$(subst v7,,$(ARCH)) - mv $(BIN)/flannel-$(subst v7,,$(ARCH)) $(BIN)/flannel + mv $(BIN)/flannel-$(subst v7,,$(ARCH)) $@ + +$(WINDOWS_BIN)/flannel.exe: + -mkdir -p $(WINDOWS_BIN) + $(CURL) -L --retry 5 https://github.com/flannel-io/cni-plugin/releases/download/$(FLANNEL_VERSION)/cni-plugin-flannel-windows-amd64-$(FLANNEL_VERSION).tgz | tar -xz -C $(WINDOWS_BIN) ./flannel-amd64.exe + mv $(WINDOWS_BIN)/flannel-amd64.exe $@ ############################################################################### # Unit Tests @@ -223,8 +255,8 @@ ci: clean mod-download build static-checks test-cni-versions image-all test-inst cd: image-all cd-common ## Build fv binary for Windows -$(BIN_WIN)/win-fv.exe: $(WINFV_SRCFILES) - $(DOCKER_RUN) -e GOOS=windows $(CALICO_BUILD) sh -c '$(GIT_CONFIG_SSH) go test ./win_tests -c -o $(BIN_WIN)/win-fv.exe' +$(WINDOWS_BIN)/win-fv.exe: $(WINFV_SRCFILES) + $(DOCKER_RUN) -e GOOS=windows $(CALICO_BUILD) sh -c '$(GIT_CONFIG_SSH) go test ./win_tests -c -o $(WINDOWS_BIN)/win-fv.exe' ############################################################################### # Release @@ -278,6 +310,18 @@ release-publish-latest: release-prereqs $(MAKE) push-images-to-registries push-manifests RELEASE=true IMAGETAG=latest RELEASE=$(RELEASE) CONFIRM=$(CONFIRM) +release-windows: release-prereqs clean-windows $(DEPLOY_CONTAINER_MARKER_WINDOWS) + for registry in $(DEV_REGISTRIES); do \ + echo Pushing Windows image to $${registry}; \ + manifest_image="$${registry}/$(CNI_PLUGIN_IMAGE_WINDOWS):$(VERSION)"; \ + image_tar="$(WINDOWS_DIST)/calico-cni-plugin-windows-$(GIT_VERSION).tar"; \ + image="$${registry}/$(CNI_PLUGIN_IMAGE_WINDOWS):$(VERSION)"; \ + $(CRANE_BINDMOUNT) push $${image_tar} $${image}$(double_quote); \ + $(DOCKER_MANIFEST) create --amend $${manifest_image} $${image}; \ + $(DOCKER_MANIFEST) annotate --os windows --arch amd64 $${manifest_image} $${image}; \ + $(DOCKER_MANIFEST) push --purge $${manifest_image}; \ + done ; + ############################################################################### # Developer helper scripts (not used by build or test) ############################################################################### diff --git a/cni-plugin/cmd/calico/calico.go b/cni-plugin/cmd/calico/calico.go index 091c080fa51..055c9f26882 100644 --- a/cni-plugin/cmd/calico/calico.go +++ b/cni-plugin/cmd/calico/calico.go @@ -36,7 +36,7 @@ func main() { plugin.Main(VERSION) case "calico-ipam", "calico-ipam.exe": ipamplugin.Main(VERSION) - case "install": + case "install", "install.exe": err := install.Install() if err != nil { logrus.WithError(err).Fatal("Error installing CNI plugin") diff --git a/cni-plugin/pkg/install/install.go b/cni-plugin/pkg/install/install.go index 31fe35a6cc3..9051963ee48 100644 --- a/cni-plugin/pkg/install/install.go +++ b/cni-plugin/pkg/install/install.go @@ -22,6 +22,8 @@ import ( "io" "os" "os/exec" + "path/filepath" + "runtime" "strings" "time" @@ -118,6 +120,20 @@ func loadConfig() config { return c } +// getServiceAccountFilePaths returns the mount paths for a container +// In the case of Windows HostProcess containers this prepends the CONTAINER_SANDBOX_MOUNT_POINT env variable +// for other operating systems or if the sandbox env variable is not set it returns the standard mount points +// see https://kubernetes.io/docs/tasks/configure-pod-container/create-hostprocess-pod/#volume-mounts +func getHostPath(path string) string { + if runtime.GOOS == "windows" { + sandbox := os.Getenv("CONTAINER_SANDBOX_MOUNT_POINT") + // join them and return with forward slashes so it can be serialized properly in json later if required + path := filepath.Join(sandbox, path) + return filepath.ToSlash(path) + } + return path +} + func Install() error { // Make sure the RNG is seeded. seedrng.EnsureSeeded() @@ -126,7 +142,7 @@ func Install() error { logrus.SetFormatter(&logutils.Formatter{Component: "cni-installer"}) // Clean up any existing binaries / config / assets. - if err := os.RemoveAll("/host/etc/cni/net.d/calico-tls"); err != nil && !os.IsNotExist(err) { + if err := os.RemoveAll(getHostPath("/host/etc/cni/net.d/calico-tls")); err != nil && !os.IsNotExist(err) { logrus.WithError(err).Warnf("Error removing old TLS directory") } @@ -136,7 +152,7 @@ func Install() error { // Determine if we're running as a Kubernetes pod. var kubecfg *rest.Config - serviceAccountTokenFile := "/var/run/secrets/kubernetes.io/serviceaccount/token" + serviceAccountTokenFile := getHostPath("/var/run/secrets/kubernetes.io/serviceaccount/token") c.ServiceAccountToken = make([]byte, 0) var err error if fileExists(serviceAccountTokenFile) { @@ -160,35 +176,35 @@ func Install() error { // First check if the dir exists and has anything in it. if directoryExists(c.TLSAssetsDir) { logrus.Info("Installing any TLS assets") - mkdir("/host/etc/cni/net.d/calico-tls") - if err := copyFileAndPermissions(fmt.Sprintf("%s/%s", c.TLSAssetsDir, "etcd-ca"), "/host/etc/cni/net.d/calico-tls/etcd-ca"); err != nil { + mkdir(getHostPath("/host/etc/cni/net.d/calico-tls")) + if err := copyFileAndPermissions(fmt.Sprintf("%s/%s", c.TLSAssetsDir, "etcd-ca"), getHostPath("/host/etc/cni/net.d/calico-tls/etcd-ca")); err != nil { logrus.Warnf("Missing etcd-ca") } - if err := copyFileAndPermissions(fmt.Sprintf("%s/%s", c.TLSAssetsDir, "etcd-cert"), "/host/etc/cni/net.d/calico-tls/etcd-cert"); err != nil { + if err := copyFileAndPermissions(fmt.Sprintf("%s/%s", c.TLSAssetsDir, "etcd-cert"), getHostPath("/host/etc/cni/net.d/calico-tls/etcd-cert")); err != nil { logrus.Warnf("Missing etcd-cert") } - if err := copyFileAndPermissions(fmt.Sprintf("%s/%s", c.TLSAssetsDir, "etcd-key"), "/host/etc/cni/net.d/calico-tls/etcd-key"); err != nil { + if err := copyFileAndPermissions(fmt.Sprintf("%s/%s", c.TLSAssetsDir, "etcd-key"), getHostPath("/host/etc/cni/net.d/calico-tls/etcd-key")); err != nil { logrus.Warnf("Missing etcd-key") } } // Set the suid bit on the binaries to allow them to run as non-root users. - if err := setSuidBit("/opt/cni/bin/install"); err != nil { + if err := setSuidBit(getHostPath("/opt/cni/bin/install")); err != nil { logrus.WithError(err).Fatalf("Failed to set the suid bit on the install binary") } // TODO: Remove the setSUID code here on calico and calico-ipam when they eventually // get refactored to all use install as the source. - if err := setSuidBit("/opt/cni/bin/calico"); err != nil { + if err := setSuidBit(getHostPath("/opt/cni/bin/calico")); err != nil { logrus.WithError(err).Fatalf("Failed to set the suid bit on the calico binary") } - if err := setSuidBit("/opt/cni/bin/calico-ipam"); err != nil { + if err := setSuidBit(getHostPath("/opt/cni/bin/calico-ipam")); err != nil { logrus.WithError(err).Fatalf("Failed to set the suid bit on the calico-ipam") } // Place the new binaries if the directory is writeable. - dirs := []string{"/host/opt/cni/bin", "/host/secondary-bin-dir"} + dirs := []string{getHostPath("/host/opt/cni/bin"), getHostPath("/host/secondary-bin-dir")} binsWritten := false for _, d := range dirs { if err := fileutil.IsDirWriteable(d); err != nil { @@ -197,13 +213,13 @@ func Install() error { } // Iterate through each binary we might want to install. - files, err := os.ReadDir("/opt/cni/bin/") + files, err := os.ReadDir(getHostPath("/opt/cni/bin/")) if err != nil { logrus.Fatal(err) } for _, binary := range files { target := fmt.Sprintf("%s/%s", d, binary.Name()) - source := fmt.Sprintf("/opt/cni/bin/%s", binary.Name()) + source := getHostPath(fmt.Sprintf("/opt/cni/bin/%s", binary.Name())) if c.skipBinary(binary.Name()) { continue } @@ -276,7 +292,7 @@ func Install() error { logrus.Warn(err) } for _, f := range files { - if err = copyFileAndPermissions(c.TLSAssetsDir+"/"+f.Name(), "/host/etc/cni/net.d/calico-tls/"+f.Name()); err != nil { + if err = copyFileAndPermissions(c.TLSAssetsDir+"/"+f.Name(), getHostPath("/host/etc/cni/net.d/calico-tls/"+f.Name())); err != nil { logrus.Warn(err) continue } @@ -305,28 +321,7 @@ func isValidJSON(s string) error { } func writeCNIConfig(c config) { - netconf := `{ - "name": "k8s-pod-network", - "cniVersion": "0.3.1", - "plugins": [ - { - "type": "calico", - "log_level": "__LOG_LEVEL__", - "log_file_path": "__LOG_FILE_PATH__", - "datastore_type": "__DATASTORE_TYPE__", - "nodename": "__KUBERNETES_NODE_NAME__", - "mtu": __CNI_MTU__, - "ipam": {"type": "calico-ipam"}, - "policy": {"type": "k8s"}, - "kubernetes": {"kubeconfig": "__KUBECONFIG_FILEPATH__"} - }, - { - "type": "portmap", - "snat": true, - "capabilities": {"portMappings": true} - } - ] -}` + netconf := defaultNetConf() // Pick the config template to use. This can either be through an env var, // or a file mounted into the container. @@ -346,11 +341,15 @@ func writeCNIConfig(c config) { kubeconfigPath := c.CNINetDir + "/calico-kubeconfig" - // Perform replacements of variables. nodename, err := names.Hostname() if err != nil { logrus.Fatal(err) } + + // Perform replacement of platform specific variables + netconf = replacePlatformSpecificVars(c, netconf) + + // Perform replacements of variables. netconf = strings.Replace(netconf, "__LOG_LEVEL__", getEnv("LOG_LEVEL", "info"), -1) netconf = strings.Replace(netconf, "__LOG_FILE_PATH__", getEnv("LOG_FILE_PATH", "/var/log/calico/cni/cni.log"), -1) netconf = strings.Replace(netconf, "__LOG_FILE_MAX_SIZE__", getEnv("LOG_FILE_MAX_SIZE", "100"), -1) @@ -368,21 +367,21 @@ func writeCNIConfig(c config) { // Replace etcd datastore variables. hostSecretsDir := c.CNINetDir + "/calico-tls" - if fileExists("/host/etc/cni/net.d/calico-tls/etcd-cert") { + if fileExists(getHostPath("/host/etc/cni/net.d/calico-tls/etcd-cert")) { etcdCertFile := fmt.Sprintf("%s/etcd-cert", hostSecretsDir) netconf = strings.Replace(netconf, "__ETCD_CERT_FILE__", etcdCertFile, -1) } else { netconf = strings.Replace(netconf, "__ETCD_CERT_FILE__", "", -1) } - if fileExists("/host/etc/cni/net.d/calico-tls/etcd-ca") { + if fileExists(getHostPath("/host/etc/cni/net.d/calico-tls/etcd-ca")) { etcdCACertFile := fmt.Sprintf("%s/etcd-ca", hostSecretsDir) netconf = strings.Replace(netconf, "__ETCD_CA_CERT_FILE__", etcdCACertFile, -1) } else { netconf = strings.Replace(netconf, "__ETCD_CA_CERT_FILE__", "", -1) } - if fileExists("/host/etc/cni/net.d/calico-tls/etcd-key") { + if fileExists(getHostPath("/host/etc/cni/net.d/calico-tls/etcd-key")) { etcdKeyFile := fmt.Sprintf("%s/etcd-key", hostSecretsDir) netconf = strings.Replace(netconf, "__ETCD_KEY_FILE__", etcdKeyFile, -1) } else { @@ -398,7 +397,7 @@ func writeCNIConfig(c config) { // Write out the file. name := getEnv("CNI_CONF_NAME", "10-calico.conflist") - path := fmt.Sprintf("/host/etc/cni/net.d/%s", name) + path := getHostPath(fmt.Sprintf("/host/etc/cni/net.d/%s", name)) err = os.WriteFile(path, []byte(netconf), 0644) if err != nil { logrus.Fatal(err) @@ -408,15 +407,15 @@ func writeCNIConfig(c config) { if err != nil { logrus.Fatal(err) } - logrus.Infof("Created /host/etc/cni/net.d/%s", name) + logrus.Infof("Created %s", getHostPath(fmt.Sprintf("/host/etc/cni/net.d/%s", name))) text := string(content) fmt.Println(text) // Remove any old config file, if one exists. oldName := getEnv("CNI_OLD_CONF_NAME", "10-calico.conflist") if name != oldName { - logrus.Infof("Removing /host/etc/cni/net.d/%s", oldName) - if err := os.Remove(fmt.Sprintf("/host/etc/cni/net.d/%s", oldName)); err != nil { + logrus.Infof("Removing %s", getHostPath(fmt.Sprintf("/host/etc/cni/net.d/%s", oldName))) + if err := os.Remove(getHostPath(fmt.Sprintf("/host/etc/cni/net.d/%s", oldName))); err != nil { logrus.WithError(err).Warnf("Failed to remove %s", oldName) } } @@ -447,6 +446,11 @@ func copyFileAndPermissions(src, dst string) (err error) { return fmt.Errorf("failed to rename file: %s", err) } + if runtime.GOOS == "windows" { + // chmod doesn't work on windows + return + } + // chmod the dst file to match the original permissions. si, err := os.Stat(src) if err != nil { @@ -502,12 +506,16 @@ current-context: calico-context` data = strings.Replace(data, "__TLS_CFG__", ca, -1) } - if err := os.WriteFile("/host/etc/cni/net.d/calico-kubeconfig", []byte(data), 0600); err != nil { + if err := os.WriteFile(getHostPath("/host/etc/cni/net.d/calico-kubeconfig"), []byte(data), 0600); err != nil { logrus.Fatal(err) } } func setSuidBit(file string) error { + if runtime.GOOS == "windows" { + // chmod doesn't work on windows + return nil + } fi, err := os.Stat(file) if err != nil { return fmt.Errorf("failed to stat file: %s", err) diff --git a/cni-plugin/pkg/install/install_linux.go b/cni-plugin/pkg/install/install_linux.go new file mode 100644 index 00000000000..97e55ccfae2 --- /dev/null +++ b/cni-plugin/pkg/install/install_linux.go @@ -0,0 +1,48 @@ +//go:build !windows + +// Copyright (c) 2020 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package install + +func defaultNetConf() string { + netconf := `{ + "name": "k8s-pod-network", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "calico", + "log_level": "__LOG_LEVEL__", + "log_file_path": "__LOG_FILE_PATH__", + "datastore_type": "__DATASTORE_TYPE__", + "nodename": "__KUBERNETES_NODE_NAME__", + "mtu": __CNI_MTU__, + "ipam": {"type": "calico-ipam"}, + "policy": {"type": "k8s"}, + "kubernetes": {"kubeconfig": "__KUBECONFIG_FILEPATH__"} + }, + { + "type": "portmap", + "snat": true, + "capabilities": {"portMappings": true} + } + ] +}` + return netconf +} + +// This function is currently a no-op for Linux +func replacePlatformSpecificVars(c config, netconf string) string { + return netconf +} diff --git a/cni-plugin/pkg/install/install_windows.go b/cni-plugin/pkg/install/install_windows.go new file mode 100644 index 00000000000..dbc4f493fdd --- /dev/null +++ b/cni-plugin/pkg/install/install_windows.go @@ -0,0 +1,163 @@ +//go:build windows + +// Copyright (c) 2020 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package install + +import ( + "fmt" + "strconv" + "strings" + + "github.com/sirupsen/logrus" + + "github.com/projectcalico/calico/libcalico-go/lib/winutils" +) + +func defaultNetConf() string { + netconf := `{ + "name": "k8s-pod-network", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "calico", + "name": "Calico", + "windows_use_single_network": true, + "mode": "__MODE__", + "vxlan_mac_prefix": "__MAC_PREFIX__", + "vxlan_vni": __VNI__, + "mtu": __CNI_MTU__, + "policy": { + "type": "k8s" + }, + "log_level": "__LOG_LEVEL__", + "log_file_path": "__LOG_FILE_PATH__", + "windows_loopback_DSR": __DSR_SUPPORT__, + "capabilities": {"dns": true}, + "DNS": { + "Nameservers": [__KUBERNETES_DNS_SERVERS__], + "Search": [ + "svc.cluster.local" + ] + }, + "nodename": "__KUBERNETES_NODE_NAME__", + "nodename_file": "__NODENAME_FILE__", + "nodename_file_optional": true, + "datastore_type": "__DATASTORE_TYPE__", + + "etcd_endpoints": "__ETCD_ENDPOINTS__", + "etcd_key_file": "__ETCD_KEY_FILE__", + "etcd_cert_file": "__ETCD_CERT_FILE__", + "etcd_ca_cert_file": "__ETCD_CA_CERT_FILE__", + + "kubernetes": { + "kubeconfig": "__KUBECONFIG_FILEPATH__" + }, + "ipam": { + "type": "__IPAM_TYPE__", + "subnet": "usePodCidr" + }, + + "policies": [ + { + "Name": "EndpointPolicy", + "Value": { + "Type": "OutBoundNAT", + "ExceptionList": [ + "__KUBERNETES_SERVICE_CIDR__" + ] + } + }, + { + "Name": "EndpointPolicy", + "Value": { + "Type": "__ROUTE_TYPE__", + "DestinationPrefix": "__KUBERNETES_SERVICE_CIDR__", + "NeedEncap": true + } + } + ] + } + ] +}` + return netconf +} + +// Perform replacement of windows variables +func replacePlatformSpecificVars(c config, netconf string) string { + + kubeconfigPath := "c:/" + c.CNINetDir + "/calico-kubeconfig" + netconf = strings.Replace(netconf, "__KUBECONFIG_FILEPATH__", kubeconfigPath, -1) + + netconf = strings.Replace(netconf, "__LOG_FILE_PATH__", getEnv("LOG_FILE_PATH", "c:/var/log/calico/cni/cni.log"), -1) + netconf = strings.Replace(netconf, "__ROUTE_TYPE__", getEnv("ROUTE_TYPE", "SDNROUTE"), -1) + netconf = strings.Replace(netconf, "__KUBERNETES_SERVICE_CIDR__", getEnv("KUBERNETES_SERVICE_CIDR", "10.96.0.0/12"), -1) + netconf = strings.Replace(netconf, "__VNI__", getEnv("VXLAN_VNI", "4096"), -1) + netconf = strings.Replace(netconf, "__MAC_PREFIX__", getEnv("MAC_PREFIX", "0E-2A"), -1) + + netconf = strings.Replace(netconf, "__NODENAME_FILE__", getEnv("CALICO_NODENAME_FILE", "c:/var/run/calico/nodename"), -1) + + dnsIPs := getEnv("KUBERNETES_DNS_SERVERS", "10.96.0.10") + dnsIPList := []string{} + for _, ip := range strings.Split(dnsIPs, ",") { + dnsIPList = append(dnsIPList, fmt.Sprintf("\"%s\"", strings.TrimSpace(ip))) + } + quotedDNSIPs := strings.Join(dnsIPList, ",") + netconf = strings.Replace(netconf, "__KUBERNETES_DNS_SERVERS__", quotedDNSIPs, -1) + + backend := getEnv("CALICO_NETWORKING_BACKEND", "vxlan") + if strings.ToLower(backend) == "bird" || strings.ToLower(backend) == "bgp" { + backend = "windows-bgp" + } + netconf = strings.Replace(netconf, "__MODE__", backend, -1) + + ipamType := getEnv("CNI_IPAM_TYPE", "calico-ipam") + if backend == "vxlan" && ipamType != "calico-ipam" { + logrus.Fatalf("Calico VXLAN requires IPAM type calico-ipam, not %s", ipamType) + + } + netconf = strings.Replace(netconf, "__IPAM_TYPE__", ipamType, -1) + + stdout, stderr, err := winutils.Powershell("Get-ComputerInfo | select WindowsVersion, OsBuildNumber, OsHardwareAbstractionLayer") + if err != nil { + logrus.Fatalf("Failed to interact with powershell\nerror: %s\nstderr: %s", err, stderr) + } + line := strings.Split(stdout, "\r\n")[3] + fields := strings.Fields(line) + winVer := fields[0] + winVerInt, err := strconv.Atoi(winVer) + if err != nil { + logrus.Fatalf("Error converting winVer to int\nerror: %s", err) + } + buildNum := fields[1] + buildNumInt, err := strconv.Atoi(buildNum) + if err != nil { + logrus.Fatalf("Error converting buildNum to int\nerror: %s", err) + } + hal := fields[2] + halVer := strings.Split(hal, ".")[3] + halVerInt, err := strconv.Atoi(halVer) + if err != nil { + logrus.Fatalf("Error converting halVer to int\nerror: %s", err) + } + supportsDSR := (winVerInt == 1809 && buildNumInt >= 17763 && halVerInt >= 1432) || (winVerInt >= 1903 && buildNumInt >= 18317) + if supportsDSR { + netconf = strings.Replace(netconf, "__DSR_SUPPORT__", "true", -1) + } else { + netconf = strings.Replace(netconf, "__DSR_SUPPORT__", "false", -1) + } + + return netconf +} diff --git a/lib.Makefile b/lib.Makefile index 06970b5a837..e5a63d4ee59 100644 --- a/lib.Makefile +++ b/lib.Makefile @@ -1198,6 +1198,7 @@ release-dev-image-arch-to-registry-%: $(CRANE) cp $(DEV_REGISTRY)/$(BUILD_IMAGE):$(DEV_TAG)-$* $(RELEASE_REGISTRY)/$(BUILD_IMAGE):$(RELEASE_TAG)-$*$(double_quote) # release-prereqs checks that the environment is configured properly to create a release. +.PHONY: release-prereqs release-prereqs: ifndef VERSION $(error VERSION is undefined - run using make release VERSION=vX.Y.Z) @@ -1384,3 +1385,37 @@ help: @echo "CALICO_BUILD: $(CALICO_BUILD)" @echo "-----------------------------------------------------------" +############################################################################### +# Common functions for building windows images. +############################################################################### + +CRANE_BINDMOUNT_CMD := \ + docker run --rm \ + --net=host \ + --init \ + --entrypoint /bin/sh \ + -e LOCAL_USER_ID=$(LOCAL_USER_ID) \ + -v $(CURDIR):/go/src/$(PACKAGE_NAME):rw \ + -v $(DOCKER_CONFIG):/root/.docker/config.json \ + -w /go/src/$(PACKAGE_NAME) \ + $(CALICO_BUILD) -c $(double_quote)crane + +DOCKER_MANIFEST_CMD := docker manifest + +ifdef CONFIRM +CRANE_BINDMOUNT = $(CRANE_BINDMOUNT_CMD) +DOCKER_MANIFEST = $(DOCKER_MANIFEST_CMD) +else +CRANE_BINDMOUNT = echo [DRY RUN] $(CRANE_BINDMOUNT_CMD) +DOCKER_MANIFEST = echo [DRY RUN] $(DOCKER_MANIFEST_CMD) +endif + +# Clean up the docker builder used to create Windows image tarballs. +.PHONY: clean-windows-builder +clean-windows-builder: + -docker buildx rm calico-windows-builder + +# Set up the docker builder used to create Windows image tarballs. +.PHONY: setup-windows-builder +setup-windows-builder: clean-windows-builder + docker buildx create --name=calico-windows-builder --use --platform windows/amd64 diff --git a/libcalico-go/lib/winutils/winutils.go b/libcalico-go/lib/winutils/winutils.go new file mode 100644 index 00000000000..1372304b48e --- /dev/null +++ b/libcalico-go/lib/winutils/winutils.go @@ -0,0 +1,41 @@ +// Copyright (c) 2023 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package winutils + +import ( + "bytes" + "os/exec" +) + +func Powershell(args ...string) (string, string, error) { + ps, err := exec.LookPath("powershell.exe") + if err != nil { + return "", "", err + } + + args = append([]string{"-NoProfile", "-NonInteractive"}, args...) + cmd := exec.Command(ps, args...) + + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err = cmd.Run() + if err != nil { + return "", "", err + } + + return stdout.String(), stderr.String(), err +} diff --git a/metadata.mk b/metadata.mk index 00d75fbd127..720118dd2b2 100644 --- a/metadata.mk +++ b/metadata.mk @@ -30,7 +30,10 @@ BIRD_VERSION=v0.3.3-202-g7a77fb73 # as both CI/CD and the release tooling will override this to build publishable images. DEV_REGISTRIES ?= calico -# RELEASE_REGISTIRES configures the container images registries which are published to +# RELEASE_REGISTIRES configures the container images registries which are published to # as part of an official release. # This variable is unused. Registries for releases are defined in hack/release/pkg/builder/builder.go # RELEASE_REGISTRIES = quay.io/calico docker.io/calico gcr.io/projectcalico-org eu.gcr.io/projectcalico-org asia.gcr.io/projectcalico-org us.gcr.io/projectcalico-org + +# The Windows version used as base for Calico Windows images +WINDOWS_VERSION ?= 1809 diff --git a/node/.dockerignore b/node/.dockerignore index 41e4e8c7b36..60e97bf71a8 100644 --- a/node/.dockerignore +++ b/node/.dockerignore @@ -8,3 +8,9 @@ # - CentOS repository file and our license !centos.repo !LICENSE +# - Files used in the windows image +!windows-packaging/CalicoWindows/libs +!windows-packaging/CalicoWindows/node +!windows-packaging/CalicoWindows/felix +!windows-packaging/CalicoWindows/confd +!windows-packaging/CalicoWindows/config-hpc.ps1 diff --git a/node/Dockerfile-windows b/node/Dockerfile-windows new file mode 100644 index 00000000000..3af99214008 --- /dev/null +++ b/node/Dockerfile-windows @@ -0,0 +1,44 @@ +# Copyright (c) 2023 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG WINDOWS_VERSION + +FROM mcr.microsoft.com/windows/nanoserver:${WINDOWS_VERSION} + +ARG GIT_VERSION=unknown + +# Required labels for certification +LABEL name="Calico node" \ + vendor="Project Calico" \ + version=$GIT_VERSION \ + release="1" \ + summary="Calico node handles networking and policy for Calico" \ + description="Calico node handles networking and policy for Calico" \ + maintainer="maintainers@projectcalico.org" + +ADD LICENSE /licenses/ + +ADD dist/bin/calico-node.exe /CalicoWindows/calico-node.exe +ADD windows-packaging/CalicoWindows/felix/felix-service.ps1 /CalicoWindows/felix-service.ps1 +ADD windows-packaging/CalicoWindows/node/node-service.ps1 /CalicoWindows/node-service.ps1 +ADD windows-packaging/CalicoWindows/confd /CalicoWindows/confd +ADD windows-packaging/CalicoWindows/config-hpc.ps1 /CalicoWindows/config.ps1 +Add windows-packaging/CalicoWindows/libs/calico/calico.psm1 /CalicoWindows/libs/calico/calico.psm1 +Add windows-packaging/CalicoWindows/libs/hns/hns.psm1 /CalicoWindows/libs/hns/hns.psm1 + +ENV PATH=$PATH:/CalicoWindows +WORKDIR /CalicoWindows +# The nanoserver image does not have powershell but this works because +# this container will be running on the host. +ENTRYPOINT ["powershell"] diff --git a/node/Makefile b/node/Makefile index 67f4d43ee44..a1581fdc9e8 100644 --- a/node/Makefile +++ b/node/Makefile @@ -8,7 +8,7 @@ DEV_TAG_SUFFIX ?=0.dev # Name of the images. # e.g., /: NODE_IMAGE ?=node -WINDOWS_IMAGE ?=windows +WINDOWS_IMAGE ?=node-windows WINDOWS_UPGRADE_IMAGE ?=windows-upgrade WINDOWS_VERSIONS?=1809 2004 20H2 ltsc2022 @@ -75,6 +75,7 @@ TEST_CONTAINER_FILES=$(shell find tests/ -type f ! -name '*.created') # Variables controlling the image NODE_CONTAINER_CREATED=.calico_node.created-$(ARCH) NODE_CONTAINER_FIPS_CREATED=.calico_node.created-$(ARCH)-fips +NODE_CONTAINER_WINDOWS_CREATED=.calico_node-windows.created WINDOWS_BINARY = $(NODE_CONTAINER_BIN_DIR)/calico-node.exe TOOLS_MOUNTNS_BINARY = $(NODE_CONTAINER_BIN_DIR)/mountns-$(ARCH) @@ -192,9 +193,10 @@ clean: clean-windows docker rmi $(NODE_IMAGE):latest-$(ARCH) || true docker rmi $(NODE_IMAGE):latest-fips-$(ARCH) || true docker rmi $(TEST_CONTAINER_NAME) || true + docker rmi $(addprefix $(WINDOWS_IMAGE):latest-,$(WINDOWS_VERSIONS)) || true docker rmi $(addprefix $(WINDOWS_UPGRADE_IMAGE):latest-,$(WINDOWS_VERSIONS)) || true -clean-windows: +clean-windows: clean-windows-builder -rm -f $(WINDOWS_ARCHIVE) $(WINDOWS_ARCHIVE_BINARY) $(WINDOWS_BINARY) -rm -f $(WINDOWS_ARCHIVE_ROOT)/libs/hns/hns.psm1 -rm -f $(WINDOWS_ARCHIVE_ROOT)/libs/hns/License.txt @@ -286,7 +288,7 @@ endif # Building the image ############################################################################### ## Create the images for all supported ARCHes -image-all: $(addprefix sub-image-,$(VALIDARCHES)) sub-image-fips-amd64 +image-all: $(addprefix sub-image-,$(VALIDARCHES)) sub-image-fips-amd64 image-tar-windows sub-image-%: $(MAKE) image ARCH=$* sub-image-fips-%: @@ -303,6 +305,21 @@ $(NODE_CONTAINER_FIPS_CREATED): $(REMOTE_DEPS) ./Dockerfile.$(ARCH) $(NODE_CONTA $(MAKE) retag-build-images-with-registries VALIDARCHES=$(ARCH) IMAGETAG=latest-fips LATEST_IMAGE_TAG=latest-fips touch $@ +$(NODE_CONTAINER_WINDOWS_CREATED): clean-windows setup-windows-builder Dockerfile-windows $(WINDOWS_BINARY) $(WINDOWS_ARCHIVE_ROOT)/libs/hns/hns.psm1 $(WINDOWS_ARCHIVE_ROOT)/libs/calico/calico.psm1 $(WINDOWS_ARCHIVE_ROOT)/config-hpc.ps1 $(WINDOWS_ARCHIVE_ROOT)/felix/felix-service.ps1 $(WINDOWS_ARCHIVE_ROOT)/node/node-service.ps1 + # ensure dir for windows image tars exits + -mkdir -p $(WINDOWS_DIST) + docker buildx build \ + --platform windows/amd64 \ + --output=type=docker,dest=$(CURDIR)/$(WINDOWS_DIST)/$(WINDOWS_IMAGE)-$(GIT_VERSION).tar \ + --pull \ + -t $(WINDOWS_IMAGE):latest \ + --build-arg GIT_VERSION=$(GIT_VERSION) \ + --build-arg=WINDOWS_VERSION=$(WINDOWS_VERSION) \ + -f Dockerfile-windows . + +.PHONY: image-tar-windows +image-tar-windows: $(NODE_CONTAINER_WINDOWS_CREATED) + # download BIRD source to include in image. $(BIRD_SOURCE): .bird-source.created .bird-source.created: @@ -506,7 +523,7 @@ release-build: .release-$(VERSION).created touch $@ ## Produces the Windows installation ZIP archive for the release. -release-windows-archive $(WINDOWS_ARCHIVE): release-prereqs +release-windows-archive: release-prereqs $(MAKE) build-windows-archive WINDOWS_ARCHIVE_TAG=$(VERSION) ## Verifies the release artifacts produces by `make release-build` are correct. @@ -566,7 +583,10 @@ $(WINDOWS_ARCHIVE_ROOT)/libs/hns/License.txt: windows-packaging/nssm.zip: wget -O windows-packaging/nssm.zip https://nssm.cc/ci/nssm-$(WINDOWS_NSSM_VERSION).zip -build-windows-archive: $(WINDOWS_ARCHIVE_FILES) windows-packaging/nssm.zip +$(WINDOWS_ARCHIVE): build-windows-archive + +.PHONY: build-windows-archive +build-windows-archive: build $(WINDOWS_ARCHIVE_FILES) windows-packaging/nssm.zip # To be as atomic as possible, we re-do work like unpacking NSSM here. -rm -f "$(WINDOWS_ARCHIVE)" -rm -rf $(WINDOWS_ARCHIVE_ROOT)/nssm @@ -624,37 +644,11 @@ build-windows-upgrade-archive: clean-windows $(WINDOWS_UPGRADE_INSTALL_ZIP) $(WI rm $(WINDOWS_UPGRADE_ARCHIVE) || true cd $(WINDOWS_UPGRADE_DIST_STAGE) && zip -r "$(CURDIR)/$(WINDOWS_UPGRADE_ARCHIVE)" *.zip *.ps1 -# Sets up the docker builder used to create Windows image tarballs. -setup-windows-builder: - -docker buildx rm calico-windows-builder - docker buildx create --name=calico-windows-builder --use --platform windows/amd64 - # Builds all the Windows image tarballs for each version in WINDOWS_VERSIONS image-tar-windows-all: setup-windows-builder $(addprefix sub-image-tar-windows-,$(WINDOWS_VERSIONS)) $(addprefix sub-image-tar-windows-upgrade-,$(WINDOWS_VERSIONS)) -CRANE_BINDMOUNT_CMD := \ - docker run --rm \ - --net=host \ - --init \ - --entrypoint /bin/sh \ - -e LOCAL_USER_ID=$(LOCAL_USER_ID) \ - -v $(CURDIR):/go/src/$(PACKAGE_NAME):rw \ - -v $(DOCKER_CONFIG):/root/.docker/config.json \ - -w /go/src/$(PACKAGE_NAME) \ - $(CALICO_BUILD) -c $(double_quote)crane - -DOCKER_MANIFEST_CMD := docker manifest - -ifdef CONFIRM -CRANE_BINDMOUNT = $(CRANE_BINDMOUNT_CMD) -DOCKER_MANIFEST = $(DOCKER_MANIFEST_CMD) -else -CRANE_BINDMOUNT = echo [DRY RUN] $(CRANE_BINDMOUNT_CMD) -DOCKER_MANIFEST = echo [DRY RUN] $(DOCKER_MANIFEST_CMD) -endif - # Uses the docker builder to create a Windows image tarball for a single Windows version. -sub-image-tar-windows-%: +sub-image-tar-windows-%: $(WINDOWS_ARCHIVE_BINARY) dist/calico-windows-$(GIT_VERSION).zip $(WINDOWS_INSTALL_SCRIPT) # ensure dir for windows image tars -mkdir -p $(WINDOWS_DIST) # ensure docker build dir exists @@ -733,3 +727,15 @@ endif done; \ $(DOCKER_MANIFEST) push --purge $${manifest_image}; \ done ; + +release-windows: release-prereqs clean-windows $(NODE_CONTAINER_WINDOWS_CREATED) + for registry in $(DEV_REGISTRIES); do \ + echo Pushing Windows image to $${registry}; \ + manifest_image="$${registry}/$(WINDOWS_IMAGE):$(VERSION)"; \ + image_tar="$(WINDOWS_DIST)/$(WINDOWS_IMAGE)-$(GIT_VERSION).tar"; \ + image="$${registry}/$(WINDOWS_IMAGE):$(VERSION)"; \ + $(CRANE_BINDMOUNT) push $${image_tar} $${image}$(double_quote); \ + $(DOCKER_MANIFEST) create --amend $${manifest_image} $${image}; \ + $(DOCKER_MANIFEST) annotate --os windows --arch amd64 $${manifest_image} $${image}; \ + $(DOCKER_MANIFEST) push --purge $${manifest_image}; \ + done ; diff --git a/node/pkg/cni/token_watch.go b/node/pkg/cni/token_watch.go index ad60157480c..d6e5e6bdcaa 100644 --- a/node/pkg/cni/token_watch.go +++ b/node/pkg/cni/token_watch.go @@ -7,6 +7,7 @@ import ( "fmt" "math/rand" "os" + "runtime" "strings" "sync" "time" @@ -22,6 +23,7 @@ import ( const ( defaultServiceAccountName = "calico-cni-plugin" + defaultServiceAccountNameWin = "calico-cni-plugin-windows" serviceAccountNamespace = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" tokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token" defaultCNITokenValiditySeconds = 24 * 60 * 60 @@ -237,12 +239,15 @@ func Run() { } // CNIServiceAccountName returns the name of the serviceaccount to use for the CNI plugin token request. -// This can be set via the CALICO_CNI_SERVICE_ACCOUNT environment variable, and defaults to "calico-cni-plugin" otherwise. +// This can be set via the CALICO_CNI_SERVICE_ACCOUNT environment variable, and defaults to "calico-cni-plugin" (on Linux, "calico-cni-plugin-windows" on Windows) otherwise. func CNIServiceAccountName() string { if sa := os.Getenv("CALICO_CNI_SERVICE_ACCOUNT"); sa != "" { logrus.WithField("name", sa).Debug("Using service account from CALICO_CNI_SERVICE_ACCOUNT") return sa } + if runtime.GOOS == "windows" { + return defaultServiceAccountNameWin + } return defaultServiceAccountName } diff --git a/node/pkg/winupgrade/upgrade.go b/node/pkg/winupgrade/upgrade.go index 27d68ebdb7e..34fa435c726 100644 --- a/node/pkg/winupgrade/upgrade.go +++ b/node/pkg/winupgrade/upgrade.go @@ -14,11 +14,9 @@ package winupgrade import ( - "bytes" "context" "fmt" "os" - "os/exec" "os/signal" "path/filepath" "strings" @@ -34,6 +32,7 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" "github.com/projectcalico/calico/node/pkg/lifecycle/utils" log "github.com/sirupsen/logrus" @@ -83,7 +82,7 @@ func Run() { log.WithError(err).Fatal("Failed to create Kubernetes client") } - stdout, stderr, err := powershell("Get-ComputerInfo | select WindowsVersion, OsBuildNumber, OsHardwareAbstractionLayer") + stdout, stderr, err := winutils.Powershell("Get-ComputerInfo | select WindowsVersion, OsBuildNumber, OsHardwareAbstractionLayer") fmt.Println(stdout, stderr) if err != nil { log.WithError(err).Fatal("Failed to interact with powershell") @@ -208,7 +207,7 @@ func kubeConfigFile() string { func uninstall() error { path := filepath.Join(baseDir(), "uninstall-calico.ps1") log.Infof("Start uninstall script %s\n", path) - stdout, stderr, err := powershell(path + " -ExceptUpgradeService $true") + stdout, stderr, err := winutils.Powershell(path + " -ExceptUpgradeService $true") fmt.Println(stdout, stderr) if err != nil { return err @@ -216,7 +215,7 @@ func uninstall() error { // After the uninstall completes, move the existing calico-node.exe to // a temporary file. The calico-upgrade service is still running so not // doing this means we cannot replace calico-node.exe with the upgrade. - stdout, stderr, err = powershell(fmt.Sprintf(`mv %v\calico-node.exe %v\calico-node.exe.to-be-replaced`, baseDir(), baseDir())) + stdout, stderr, err = winutils.Powershell(fmt.Sprintf(`mv %v\calico-node.exe %v\calico-node.exe.to-be-replaced`, baseDir(), baseDir())) fmt.Println(stdout, stderr) if err != nil { return err @@ -234,7 +233,7 @@ func execScript(script string) error { // process running the upgrade script is killed and the installation is left // incomplete. cmd := fmt.Sprintf(`Start-Process powershell -argument %q -WindowStyle hidden`, script) - stdout, stderr, err := powershell(cmd) + stdout, stderr, err := winutils.Powershell(cmd) if err != nil { return err @@ -387,25 +386,3 @@ func verifyPodImageWithHostPathVolume(cs kubernetes.Interface, nodeName string, } return nil } - -func powershell(args ...string) (string, string, error) { - ps, err := exec.LookPath("powershell.exe") - if err != nil { - return "", "", err - } - - args = append([]string{"-NoProfile", "-NonInteractive"}, args...) - cmd := exec.Command(ps, args...) - - var stdout bytes.Buffer - var stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - - err = cmd.Run() - if err != nil { - return "", "", err - } - - return stdout.String(), stderr.String(), err -} diff --git a/node/windows-packaging/CalicoWindows/confd/confd-service.ps1 b/node/windows-packaging/CalicoWindows/confd/confd-service.ps1 index 0994b17e720..9aa4eea45ff 100644 --- a/node/windows-packaging/CalicoWindows/confd/confd-service.ps1 +++ b/node/windows-packaging/CalicoWindows/confd/confd-service.ps1 @@ -45,4 +45,3 @@ if($env:CALICO_NETWORKING_BACKEND = "windows-bgp") Start-Sleep 10 } } - diff --git a/node/windows-packaging/CalicoWindows/config-hpc.ps1 b/node/windows-packaging/CalicoWindows/config-hpc.ps1 new file mode 100644 index 00000000000..c74dc58080a --- /dev/null +++ b/node/windows-packaging/CalicoWindows/config-hpc.ps1 @@ -0,0 +1,61 @@ +$baseDir = "$PSScriptRoot" +ipmo $baseDir\libs\calico\calico.psm1 -Force + +Write-Host "Setting environment variables if not set..." + +## Cluster configuration: + +# KUBE_NETWORK should be set to a regular expression that matches the HNS network(s) used for pods. +# The default, "Calico.*", is correct for Calico CNI. +Set-EnvVarIfNotSet -var "KUBE_NETWORK" -defaultValue "Calico.*" + +# Set this to one of the following values: +# - "vxlan" for Calico VXLAN networking +# - "windows-bgp" for Calico BGP networking using the Windows BGP router. +# - "none" to disable the Calico CNI plugin (so that you can use another plugin). +Set-EnvVarIfNotSet -var "CALICO_NETWORKING_BACKEND" -defaultValue "vxlan" + +# Set to match your Kubernetes service CIDR. +Set-EnvVarIfNotSet -var "DNS_SEARCH" -defaultValue "svc.cluster.local" + +## VXLAN-specific configuration. + +# The VXLAN VNI / VSID. Must match the VXLANVNI felix configuration parameter used +# for Linux nodes. +Set-EnvVarIfNotSet -var "VXLAN_VNI" -defaultValue 4096 +# Prefix used when generating MAC addresses for virtual NICs. +Set-EnvVarIfNotSet -var "VXLAN_MAC_PREFIX" -defaultValue "0E-2A" +# Network Adapter used on VXLAN, leave blank for primary NIC. +Set-EnvVarIfNotSet -var "VXLAN_ADAPTER" -defaultValue "" + +## Node configuration. + +# The NODENAME variable should be set to match the Kubernetes Node name of this host. +# The default uses this node's hostname (which is the same as kubelet). +# +# Note: on AWS, kubelet is often configured to use the internal domain name of the host rather than +# the simple hostname, for example "ip-172-16-101-135.us-west-2.compute.internal". +Set-EnvVarIfNotSet -var "NODENAME" -defaultValue $(hostname).ToLower() +# Similarly, CALICO_K8S_NODE_REF should be set to the Kubernetes Node name. When using etcd, +# the Calico kube-controllers pod will clean up Calico node objects if the corresponding Kubernetes Node is +# cleaned up. +Set-EnvVarIfNotSet -var "CALICO_K8S_NODE_REF" -defaultValue $env:NODENAME + +# The time out to wait for a valid IP of an interface to be assigned before initialising Calico +# after a reboot. +Set-EnvVarIfNotSet -var "STARTUP_VALID_IP_TIMEOUT" -defaultValue 90 + +# The IP of the node; the default will auto-detect a usable IP in most cases. +Set-EnvVarIfNotSet -var "IP" -defaultValue "autodetect" + +# Felix logs to screen at info level by default. Uncomment this line to override the log +# level. Alternatively, (if this is commented out) the log level can be controlled via +# the FelixConfiguration resource in the datastore. +# $env:FELIX_LOGSEVERITYSCREEN = "info" +# Disable logging to file by default since the service wrapper will redirect our log to file. +Set-EnvVarIfNotSet -var "FELIX_LOGSEVERITYFILE" -defaultValue "none" +# Disable syslog logging, which is not supported on Windows. +Set-EnvVarIfNotSet -var "FELIX_LOGSEVERITYSYS" -defaultValue "none" +# confd logs to screen at info level by default. Uncomment this line to override the log +# level. +#Set-EnvVarIfNotSet -var "BGP_LOGSEVERITYSCREEN" -defaultValue "debug" From f9da7f599feea7b2bf73fa1c879f2c89999ac765 Mon Sep 17 00:00:00 2001 From: Pedro Coutinho Date: Tue, 18 Jul 2023 14:56:17 -0700 Subject: [PATCH 02/16] Use windows-host-process-containers-base-image instead of nanoserver as base for windows HPC images. Add uninstall-calico powershell script to be run in initContainer, which will clean up non-HPC Calico installations from the host before starting Calico Windows HPC. Fix bug in confd-service.ps1 --- .gitignore | 1 + cni-plugin/Dockerfile-windows | 5 +- cni-plugin/Makefile | 50 +++++------- metadata.mk | 4 +- node/.dockerignore | 2 + node/Dockerfile-windows | 10 ++- node/Makefile | 58 +++++++------- .../CalicoWindows/confd/confd-service.ps1 | 2 +- .../CalicoWindows/config-hpc.ps1 | 14 ++++ .../CalicoWindows/config.ps1 | 14 ++++ .../CalicoWindows/uninstall-calico-hpc.ps1 | 80 +++++++++++++++++++ 11 files changed, 176 insertions(+), 64 deletions(-) create mode 100644 node/windows-packaging/CalicoWindows/uninstall-calico-hpc.ps1 diff --git a/.gitignore b/.gitignore index 0828379c129..65a58edd155 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ crypto/tmp hack/release/release node/windows-packaging/CalicoWindows/confd/config-bgp.ps1 node/windows-packaging/CalicoWindows/confd/config-bgp.psm1 +node/windows-packaging/nssm.zip _output builder.coverprofile hack/release/ghr diff --git a/cni-plugin/Dockerfile-windows b/cni-plugin/Dockerfile-windows index b2188b3afa6..d366e1e5339 100644 --- a/cni-plugin/Dockerfile-windows +++ b/cni-plugin/Dockerfile-windows @@ -12,8 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG WINDOWS_VERSION -FROM mcr.microsoft.com/windows/nanoserver:${WINDOWS_VERSION} +ARG WINDOWS_HPC_VERSION + +FROM mcr.microsoft.com/oss/kubernetes/windows-host-process-containers-base-image:${WINDOWS_HPC_VERSION} ARG GIT_VERSION=unknown diff --git a/cni-plugin/Makefile b/cni-plugin/Makefile index 8ed1ee66afb..f034d5c076a 100644 --- a/cni-plugin/Makefile +++ b/cni-plugin/Makefile @@ -39,7 +39,6 @@ CNI_SPEC_VERSION?=0.3.1 # Markers for various images we produce. DEPLOY_CONTAINER_STATIC_MARKER=cni_deploy_container-$(ARCH).created DEPLOY_CONTAINER_FIPS_MARKER=cni_deploy_container-$(ARCH)-fips.created -DEPLOY_CONTAINER_MARKER_WINDOWS=cni_deploy_container-windows.created # Set FIPS to true to enable a FIPS compliant build using BoringSSL and CGO. # Note that this produces binaries that are dynamically linked, and so have dependencies @@ -57,9 +56,6 @@ BIN=bin/$(ARCH) DEPLOY_CONTAINER_MARKER=$(DEPLOY_CONTAINER_STATIC_MARKER) endif -# The directory for windows image tarball -WINDOWS_DIST = dist/windows - .PHONY: clean clean-windows clean: clean-windows # Clean .created files which indicate images / releases have been built. @@ -71,7 +67,7 @@ clean: clean-windows rm -rf dist/ clean-windows: clean-windows-builder - rm -rf $(WINDOWS_BIN) $(WINDOWS_DIST) $(DEPLOY_CONTAINER_MARKER_WINDOWS) + rm -rf $(WINDOWS_BIN) ############################################################################### # Building the binary @@ -123,8 +119,7 @@ $(WINDOWS_BIN)/calico-ipam.exe $(WINDOWS_BIN)/calico.exe: $(WINDOWS_BIN)/install # Building the image ############################################################################### image: $(DEPLOY_CONTAINER_MARKER) -image-tar-windows: $(DEPLOY_CONTAINER_MARKER_WINDOWS) -image-all: $(addprefix sub-image-,$(VALIDARCHES)) sub-image-fips-amd64 image-tar-windows +image-all: $(addprefix sub-image-,$(VALIDARCHES)) sub-image-fips-amd64 image-windows sub-image-%: $(MAKE) image ARCH=$* sub-image-fips-%: @@ -142,18 +137,6 @@ $(DEPLOY_CONTAINER_FIPS_MARKER): Dockerfile.$(ARCH) build fetch-cni-bins $(MAKE) retag-build-images-with-registries VALIDARCHES=$(ARCH) IMAGETAG=latest-fips LATEST_IMAGE_TAG=latest-fips touch $@ -$(DEPLOY_CONTAINER_MARKER_WINDOWS): setup-windows-builder Dockerfile-windows build fetch-win-cni-bins - -mkdir -p $(WINDOWS_DIST) - docker buildx build \ - --platform windows/amd64 \ - --output=type=docker,dest=$(WINDOWS_DIST)/calico-cni-plugin-windows-$(GIT_VERSION).tar \ - -t $(CNI_PLUGIN_IMAGE_WINDOWS):latest \ - --pull \ - --no-cache \ - --build-arg GIT_VERSION=$(GIT_VERSION) \ - --build-arg WINDOWS_VERSION=$(WINDOWS_VERSION) \ - -f Dockerfile-windows . - .PHONY: fetch-cni-bins fetch-win-cni-bins fetch-cni-bins: $(BIN)/flannel $(BIN)/loopback $(BIN)/host-local $(BIN)/portmap $(BIN)/tuning $(BIN)/bandwidth fetch-win-cni-bins: $(WINDOWS_BIN)/flannel.exe @@ -310,18 +293,29 @@ release-publish-latest: release-prereqs $(MAKE) push-images-to-registries push-manifests RELEASE=true IMAGETAG=latest RELEASE=$(RELEASE) CONFIRM=$(CONFIRM) -release-windows: release-prereqs clean-windows $(DEPLOY_CONTAINER_MARKER_WINDOWS) +.PHONY: image-windows release-windows +# Build Windows image and possibly push it to $DEV_REGISTRIES +image-windows: setup-windows-builder Dockerfile-windows build fetch-win-cni-bins + push="$${PUSH:-false}"; \ + image_version="$${VERSION:-latest}"; \ for registry in $(DEV_REGISTRIES); do \ - echo Pushing Windows image to $${registry}; \ - manifest_image="$${registry}/$(CNI_PLUGIN_IMAGE_WINDOWS):$(VERSION)"; \ - image_tar="$(WINDOWS_DIST)/calico-cni-plugin-windows-$(GIT_VERSION).tar"; \ - image="$${registry}/$(CNI_PLUGIN_IMAGE_WINDOWS):$(VERSION)"; \ - $(CRANE_BINDMOUNT) push $${image_tar} $${image}$(double_quote); \ - $(DOCKER_MANIFEST) create --amend $${manifest_image} $${image}; \ - $(DOCKER_MANIFEST) annotate --os windows --arch amd64 $${manifest_image} $${image}; \ - $(DOCKER_MANIFEST) push --purge $${manifest_image}; \ + echo Building and pushing Windows image to $${registry}; \ + image="$${registry}/$(CNI_PLUGIN_IMAGE_WINDOWS):$${image_version}"; \ + docker buildx build \ + --platform windows/amd64 \ + --output=type=image,push=$* \ + -t $${image} \ + --pull \ + --no-cache \ + --build-arg GIT_VERSION=$(GIT_VERSION) \ + --build-arg WINDOWS_HPC_VERSION=$(WINDOWS_HPC_VERSION) \ + -f Dockerfile-windows .; \ done ; +# Build and push Windows image +release-windows: release-prereqs clean-windows + $(MAKE) image-windows PUSH=true + ############################################################################### # Developer helper scripts (not used by build or test) ############################################################################### diff --git a/metadata.mk b/metadata.mk index 720118dd2b2..d086bbc1c18 100644 --- a/metadata.mk +++ b/metadata.mk @@ -35,5 +35,5 @@ DEV_REGISTRIES ?= calico # This variable is unused. Registries for releases are defined in hack/release/pkg/builder/builder.go # RELEASE_REGISTRIES = quay.io/calico docker.io/calico gcr.io/projectcalico-org eu.gcr.io/projectcalico-org asia.gcr.io/projectcalico-org us.gcr.io/projectcalico-org -# The Windows version used as base for Calico Windows images -WINDOWS_VERSION ?= 1809 +# The Windows HPC container version used as base for Calico Windows images +WINDOWS_HPC_VERSION ?= v1.0.0 diff --git a/node/.dockerignore b/node/.dockerignore index 60e97bf71a8..707ac0ea867 100644 --- a/node/.dockerignore +++ b/node/.dockerignore @@ -14,3 +14,5 @@ !windows-packaging/CalicoWindows/felix !windows-packaging/CalicoWindows/confd !windows-packaging/CalicoWindows/config-hpc.ps1 +!windows-packaging/CalicoWindows/uninstall-calico-hpc.ps1 +!windows-packaging/nssm.exe diff --git a/node/Dockerfile-windows b/node/Dockerfile-windows index 3af99214008..1a7715a7d7e 100644 --- a/node/Dockerfile-windows +++ b/node/Dockerfile-windows @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG WINDOWS_VERSION +ARG WINDOWS_HPC_VERSION -FROM mcr.microsoft.com/windows/nanoserver:${WINDOWS_VERSION} +FROM mcr.microsoft.com/oss/kubernetes/windows-host-process-containers-base-image:${WINDOWS_HPC_VERSION} ARG GIT_VERSION=unknown @@ -34,8 +34,10 @@ ADD windows-packaging/CalicoWindows/felix/felix-service.ps1 /CalicoWindows/felix ADD windows-packaging/CalicoWindows/node/node-service.ps1 /CalicoWindows/node-service.ps1 ADD windows-packaging/CalicoWindows/confd /CalicoWindows/confd ADD windows-packaging/CalicoWindows/config-hpc.ps1 /CalicoWindows/config.ps1 -Add windows-packaging/CalicoWindows/libs/calico/calico.psm1 /CalicoWindows/libs/calico/calico.psm1 -Add windows-packaging/CalicoWindows/libs/hns/hns.psm1 /CalicoWindows/libs/hns/hns.psm1 +ADD windows-packaging/CalicoWindows/libs/calico/calico.psm1 /CalicoWindows/libs/calico/calico.psm1 +ADD windows-packaging/CalicoWindows/libs/hns/hns.psm1 /CalicoWindows/libs/hns/hns.psm1 +ADD windows-packaging/nssm.exe /nssm.exe +ADD windows-packaging/CalicoWindows/uninstall-calico-hpc.ps1 /uninstall-calico.ps1 ENV PATH=$PATH:/CalicoWindows WORKDIR /CalicoWindows diff --git a/node/Makefile b/node/Makefile index a1581fdc9e8..89064feaa30 100644 --- a/node/Makefile +++ b/node/Makefile @@ -202,6 +202,7 @@ clean-windows: clean-windows-builder -rm -f $(WINDOWS_ARCHIVE_ROOT)/libs/hns/License.txt -rm -f $(WINDOWS_ARCHIVE_ROOT)/cni/*.exe -rm -f $(WINDOWS_ARCHIVE_ROOT)/../nssm.zip + -rm -f $(WINDOWS_ARCHIVE_ROOT)/../nssm.exe -rm -f $(WINDOWS_INSTALL_SCRIPT) -rm -rf "$(WINDOWS_DIST)" -rm -rf "$(WINDOWS_UPGRADE_DIST)" @@ -288,7 +289,7 @@ endif # Building the image ############################################################################### ## Create the images for all supported ARCHes -image-all: $(addprefix sub-image-,$(VALIDARCHES)) sub-image-fips-amd64 image-tar-windows +image-all: $(addprefix sub-image-,$(VALIDARCHES)) sub-image-fips-amd64 image-windows sub-image-%: $(MAKE) image ARCH=$* sub-image-fips-%: @@ -305,21 +306,6 @@ $(NODE_CONTAINER_FIPS_CREATED): $(REMOTE_DEPS) ./Dockerfile.$(ARCH) $(NODE_CONTA $(MAKE) retag-build-images-with-registries VALIDARCHES=$(ARCH) IMAGETAG=latest-fips LATEST_IMAGE_TAG=latest-fips touch $@ -$(NODE_CONTAINER_WINDOWS_CREATED): clean-windows setup-windows-builder Dockerfile-windows $(WINDOWS_BINARY) $(WINDOWS_ARCHIVE_ROOT)/libs/hns/hns.psm1 $(WINDOWS_ARCHIVE_ROOT)/libs/calico/calico.psm1 $(WINDOWS_ARCHIVE_ROOT)/config-hpc.ps1 $(WINDOWS_ARCHIVE_ROOT)/felix/felix-service.ps1 $(WINDOWS_ARCHIVE_ROOT)/node/node-service.ps1 - # ensure dir for windows image tars exits - -mkdir -p $(WINDOWS_DIST) - docker buildx build \ - --platform windows/amd64 \ - --output=type=docker,dest=$(CURDIR)/$(WINDOWS_DIST)/$(WINDOWS_IMAGE)-$(GIT_VERSION).tar \ - --pull \ - -t $(WINDOWS_IMAGE):latest \ - --build-arg GIT_VERSION=$(GIT_VERSION) \ - --build-arg=WINDOWS_VERSION=$(WINDOWS_VERSION) \ - -f Dockerfile-windows . - -.PHONY: image-tar-windows -image-tar-windows: $(NODE_CONTAINER_WINDOWS_CREATED) - # download BIRD source to include in image. $(BIRD_SOURCE): .bird-source.created .bird-source.created: @@ -492,10 +478,10 @@ st: $(REMOTE_DEPS) image dist/calicoctl busybox.tar calico-node.tar workload.tar # CI/CD ############################################################################### .PHONY: ci -ci: static-checks ut image build-windows-upgrade-archive image-tar-windows-all st +ci: static-checks ut image image-windows st ## Deploys images to registry -cd: cd-common cd-windows-all +cd: cd-common release-windows check-boring-ssl: $(NODE_CONTAINER_BIN_DIR)/calico-node-amd64 $(DOCKER_RUN) -e CGO_ENABLED=$(CGO_ENABLED) $(CALICO_BUILD) \ @@ -583,6 +569,13 @@ $(WINDOWS_ARCHIVE_ROOT)/libs/hns/License.txt: windows-packaging/nssm.zip: wget -O windows-packaging/nssm.zip https://nssm.cc/ci/nssm-$(WINDOWS_NSSM_VERSION).zip +windows-packaging/nssm.exe: windows-packaging/nssm.zip + cd windows-packaging && \ + sha256sum --check nssm.sha256sum && \ + unzip -o nssm.zip 'nssm-$(WINDOWS_NSSM_VERSION)/win64/nssm.exe'&& \ + mv nssm-$(WINDOWS_NSSM_VERSION)/win64/nssm.exe nssm.exe && \ + rm -rf nssm-$(WINDOWS_NSSM_VERSION)/ + $(WINDOWS_ARCHIVE): build-windows-archive .PHONY: build-windows-archive @@ -728,14 +721,25 @@ endif $(DOCKER_MANIFEST) push --purge $${manifest_image}; \ done ; -release-windows: release-prereqs clean-windows $(NODE_CONTAINER_WINDOWS_CREATED) +.PHONY: image-windows release-windows +# Build Windows image and possibly push it to $DEV_REGISTRIES +image-windows: setup-windows-builder Dockerfile-windows $(WINDOWS_BINARY) $(WINDOWS_ARCHIVE_ROOT)/libs/hns/hns.psm1 $(WINDOWS_ARCHIVE_ROOT)/libs/calico/calico.psm1 $(WINDOWS_ARCHIVE_ROOT)/config-hpc.ps1 $(WINDOWS_ARCHIVE_ROOT)/felix/felix-service.ps1 $(WINDOWS_ARCHIVE_ROOT)/node/node-service.ps1 $(WINDOWS_ARCHIVE_ROOT)/uninstall-calico-hpc.ps1 windows-packaging/nssm.exe + push="$${PUSH:-false}"; \ + image_version="$${VERSION:-latest}"; \ for registry in $(DEV_REGISTRIES); do \ - echo Pushing Windows image to $${registry}; \ - manifest_image="$${registry}/$(WINDOWS_IMAGE):$(VERSION)"; \ - image_tar="$(WINDOWS_DIST)/$(WINDOWS_IMAGE)-$(GIT_VERSION).tar"; \ - image="$${registry}/$(WINDOWS_IMAGE):$(VERSION)"; \ - $(CRANE_BINDMOUNT) push $${image_tar} $${image}$(double_quote); \ - $(DOCKER_MANIFEST) create --amend $${manifest_image} $${image}; \ - $(DOCKER_MANIFEST) annotate --os windows --arch amd64 $${manifest_image} $${image}; \ - $(DOCKER_MANIFEST) push --purge $${manifest_image}; \ + echo Building and pushing Windows image to $${registry}; \ + image="$${registry}/$(WINDOWS_IMAGE):$${image_version}"; \ + docker buildx build \ + --platform windows/amd64 \ + --output=type=image,push=$${push} \ + -t $${image} \ + --pull \ + --no-cache \ + --build-arg GIT_VERSION=$(GIT_VERSION) \ + --build-arg WINDOWS_HPC_VERSION=$(WINDOWS_HPC_VERSION) \ + -f Dockerfile-windows .; \ done ; + +# Build and push Windows image +release-windows: release-prereqs clean-windows + $(MAKE) image-windows PUSH=true diff --git a/node/windows-packaging/CalicoWindows/confd/confd-service.ps1 b/node/windows-packaging/CalicoWindows/confd/confd-service.ps1 index 9aa4eea45ff..2988c8358e6 100644 --- a/node/windows-packaging/CalicoWindows/confd/confd-service.ps1 +++ b/node/windows-packaging/CalicoWindows/confd/confd-service.ps1 @@ -24,7 +24,7 @@ if ($env:CNI_IPAM_TYPE -EQ "host-local") { $env:USE_POD_CIDR = "false" } -if($env:CALICO_NETWORKING_BACKEND = "windows-bgp") +if($env:CALICO_NETWORKING_BACKEND -EQ "windows-bgp") { Wait-ForCalicoInit Write-Host "Windows BGP is enabled, running confd..." diff --git a/node/windows-packaging/CalicoWindows/config-hpc.ps1 b/node/windows-packaging/CalicoWindows/config-hpc.ps1 index c74dc58080a..67519dde9f8 100644 --- a/node/windows-packaging/CalicoWindows/config-hpc.ps1 +++ b/node/windows-packaging/CalicoWindows/config-hpc.ps1 @@ -1,3 +1,17 @@ +# Copyright (c) 2023 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + $baseDir = "$PSScriptRoot" ipmo $baseDir\libs\calico\calico.psm1 -Force diff --git a/node/windows-packaging/CalicoWindows/config.ps1 b/node/windows-packaging/CalicoWindows/config.ps1 index 4597c0360e5..80203067379 100644 --- a/node/windows-packaging/CalicoWindows/config.ps1 +++ b/node/windows-packaging/CalicoWindows/config.ps1 @@ -1,3 +1,17 @@ +# Copyright (c) 2021-2023 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + $baseDir = "$PSScriptRoot" ipmo $baseDir\libs\calico\calico.psm1 -Force diff --git a/node/windows-packaging/CalicoWindows/uninstall-calico-hpc.ps1 b/node/windows-packaging/CalicoWindows/uninstall-calico-hpc.ps1 new file mode 100644 index 00000000000..aad64d756b9 --- /dev/null +++ b/node/windows-packaging/CalicoWindows/uninstall-calico-hpc.ps1 @@ -0,0 +1,80 @@ +# Copyright (c) 2023 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +<# +.DESCRIPTION + This script stops Calico services and fully uninstalls Calico on a Windows node. +#> + +Param( + [parameter(Mandatory = $false)] $NSSMPath="$env:CONTAINER_SANDBOX_MOUNT_POINT/nssm.exe" +) + +$ErrorActionPreference = 'SilentlyContinue' + +function Remove-CalicoService($ServiceName) +{ + $svc = Get-Service | where Name -EQ "$ServiceName" + if ($svc -NE $null) + { + if ($svc.Status -EQ 'Running') + { + Write-Host "$ServiceName service is running, stopping it..." + & $NSSMPath stop $ServiceName confirm + } + Write-Host "Removing $ServiceName service..." + & $NSSMPath remove $ServiceName confirm + } +} + +if (-not (Test-Path "env:$CNI_NET_DIR")) +{ + Write-Host "CNI_NET_DIR env var not set, skipping Calico CNI config cleanup" +} else +{ + $cniConfFile = "$env:CONTAINER_SANDBOX_MOUNT_POINT/host/$env:CNI_NET_DIR/10-calico.conf" + if (Test-Path $cniConfFile) { + Write-Host "Removing Calico CNI conf file at $cniConfFile ..." + rm $cniConfFile + } + + if (Test-Path "env:$CNI_CONF_NAME") { + $cniConfListFile = "$env:CONTAINER_SANDBOX_MOUNT_POINT/host/$env:CNI_NET_DIR/$env:CNI_CONF_NAME" + if (Test-Path $cniConfListFile) { + Write-Host "Removing Calico CNI conf file at $cniConfListFile ..." + rm $cniConfListFile + } + } +} + +if (-not (Test-Path "env:$CNI_BIN_DIR")) +{ + Write-Host "CNI_BIN_DIR env var not set, skipping Calico CNI binary cleanup" +} else +{ + $cniBinPath = "$env:CONTAINER_SANDBOX_MOUNT_POINT/host/$env:CNI_BIN_DIR/calico*.exe" + if (Test-Path $cniBinPath) { + Write-Host "Removing Calico CNI binaries at $cniBinPath ..." + rm $cniBinPath + } +} + +Write-Host "Stopping and removing Calico services if they are present..." +Remove-CalicoService CalicoConfd +Remove-CalicoService CalicoFelix +Remove-CalicoService CalicoNode +Remove-CalicoService CalicoUpgrade + +Get-Module 'calico' | Remove-Module -Force +Write-Host "Done." From 1a6e1be445f776799e4359015e089de4e7d0c017 Mon Sep 17 00:00:00 2001 From: Pedro Coutinho Date: Tue, 18 Jul 2023 17:52:36 -0700 Subject: [PATCH 03/16] Remove getHostPath() function as it is no longer necessary for Windows HPC when using containerd v1.7+ --- cni-plugin/pkg/install/install.go | 63 +++++++++++++------------------ 1 file changed, 27 insertions(+), 36 deletions(-) diff --git a/cni-plugin/pkg/install/install.go b/cni-plugin/pkg/install/install.go index 9051963ee48..aac746da362 100644 --- a/cni-plugin/pkg/install/install.go +++ b/cni-plugin/pkg/install/install.go @@ -22,7 +22,6 @@ import ( "io" "os" "os/exec" - "path/filepath" "runtime" "strings" "time" @@ -120,20 +119,6 @@ func loadConfig() config { return c } -// getServiceAccountFilePaths returns the mount paths for a container -// In the case of Windows HostProcess containers this prepends the CONTAINER_SANDBOX_MOUNT_POINT env variable -// for other operating systems or if the sandbox env variable is not set it returns the standard mount points -// see https://kubernetes.io/docs/tasks/configure-pod-container/create-hostprocess-pod/#volume-mounts -func getHostPath(path string) string { - if runtime.GOOS == "windows" { - sandbox := os.Getenv("CONTAINER_SANDBOX_MOUNT_POINT") - // join them and return with forward slashes so it can be serialized properly in json later if required - path := filepath.Join(sandbox, path) - return filepath.ToSlash(path) - } - return path -} - func Install() error { // Make sure the RNG is seeded. seedrng.EnsureSeeded() @@ -142,7 +127,7 @@ func Install() error { logrus.SetFormatter(&logutils.Formatter{Component: "cni-installer"}) // Clean up any existing binaries / config / assets. - if err := os.RemoveAll(getHostPath("/host/etc/cni/net.d/calico-tls")); err != nil && !os.IsNotExist(err) { + if err := os.RemoveAll("/host/etc/cni/net.d/calico-tls"); err != nil && !os.IsNotExist(err) { logrus.WithError(err).Warnf("Error removing old TLS directory") } @@ -152,7 +137,7 @@ func Install() error { // Determine if we're running as a Kubernetes pod. var kubecfg *rest.Config - serviceAccountTokenFile := getHostPath("/var/run/secrets/kubernetes.io/serviceaccount/token") + serviceAccountTokenFile := "/var/run/secrets/kubernetes.io/serviceaccount/token" c.ServiceAccountToken = make([]byte, 0) var err error if fileExists(serviceAccountTokenFile) { @@ -176,35 +161,35 @@ func Install() error { // First check if the dir exists and has anything in it. if directoryExists(c.TLSAssetsDir) { logrus.Info("Installing any TLS assets") - mkdir(getHostPath("/host/etc/cni/net.d/calico-tls")) - if err := copyFileAndPermissions(fmt.Sprintf("%s/%s", c.TLSAssetsDir, "etcd-ca"), getHostPath("/host/etc/cni/net.d/calico-tls/etcd-ca")); err != nil { + mkdir("/host/etc/cni/net.d/calico-tls") + if err := copyFileAndPermissions(fmt.Sprintf("%s/%s", c.TLSAssetsDir, "etcd-ca"), "/host/etc/cni/net.d/calico-tls/etcd-ca"); err != nil { logrus.Warnf("Missing etcd-ca") } - if err := copyFileAndPermissions(fmt.Sprintf("%s/%s", c.TLSAssetsDir, "etcd-cert"), getHostPath("/host/etc/cni/net.d/calico-tls/etcd-cert")); err != nil { + if err := copyFileAndPermissions(fmt.Sprintf("%s/%s", c.TLSAssetsDir, "etcd-cert"), "/host/etc/cni/net.d/calico-tls/etcd-cert"); err != nil { logrus.Warnf("Missing etcd-cert") } - if err := copyFileAndPermissions(fmt.Sprintf("%s/%s", c.TLSAssetsDir, "etcd-key"), getHostPath("/host/etc/cni/net.d/calico-tls/etcd-key")); err != nil { + if err := copyFileAndPermissions(fmt.Sprintf("%s/%s", c.TLSAssetsDir, "etcd-key"), "/host/etc/cni/net.d/calico-tls/etcd-key"); err != nil { logrus.Warnf("Missing etcd-key") } } // Set the suid bit on the binaries to allow them to run as non-root users. - if err := setSuidBit(getHostPath("/opt/cni/bin/install")); err != nil { + if err := setSuidBit("/opt/cni/bin/install"); err != nil { logrus.WithError(err).Fatalf("Failed to set the suid bit on the install binary") } // TODO: Remove the setSUID code here on calico and calico-ipam when they eventually // get refactored to all use install as the source. - if err := setSuidBit(getHostPath("/opt/cni/bin/calico")); err != nil { + if err := setSuidBit("/opt/cni/bin/calico"); err != nil { logrus.WithError(err).Fatalf("Failed to set the suid bit on the calico binary") } - if err := setSuidBit(getHostPath("/opt/cni/bin/calico-ipam")); err != nil { + if err := setSuidBit("/opt/cni/bin/calico-ipam"); err != nil { logrus.WithError(err).Fatalf("Failed to set the suid bit on the calico-ipam") } // Place the new binaries if the directory is writeable. - dirs := []string{getHostPath("/host/opt/cni/bin"), getHostPath("/host/secondary-bin-dir")} + dirs := []string{"/host/opt/cni/bin", "/host/secondary-bin-dir"} binsWritten := false for _, d := range dirs { if err := fileutil.IsDirWriteable(d); err != nil { @@ -212,14 +197,20 @@ func Install() error { continue } + containerBinDir := "/opt/cni/bin/" + // The binaries dir in the container needs to be prepended by the CONTAINER_SANDBOX_MOUNT_POINT env var on Windows Host Process Containers + // see https://kubernetes.io/docs/tasks/configure-pod-container/create-hostprocess-pod/#containerd-v1-7-and-greater + if runtime.GOOS == "windows" { + containerBinDir = os.Getenv("CONTAINER_SANDBOX_MOUNT_POINT") + "/" + containerBinDir + } // Iterate through each binary we might want to install. - files, err := os.ReadDir(getHostPath("/opt/cni/bin/")) + files, err := os.ReadDir(containerBinDir) if err != nil { logrus.Fatal(err) } for _, binary := range files { target := fmt.Sprintf("%s/%s", d, binary.Name()) - source := getHostPath(fmt.Sprintf("/opt/cni/bin/%s", binary.Name())) + source := fmt.Sprintf("%s/%s", containerBinDir, binary.Name()) if c.skipBinary(binary.Name()) { continue } @@ -292,7 +283,7 @@ func Install() error { logrus.Warn(err) } for _, f := range files { - if err = copyFileAndPermissions(c.TLSAssetsDir+"/"+f.Name(), getHostPath("/host/etc/cni/net.d/calico-tls/"+f.Name())); err != nil { + if err = copyFileAndPermissions(c.TLSAssetsDir+"/"+f.Name(), "/host/etc/cni/net.d/calico-tls/"+f.Name()); err != nil { logrus.Warn(err) continue } @@ -367,21 +358,21 @@ func writeCNIConfig(c config) { // Replace etcd datastore variables. hostSecretsDir := c.CNINetDir + "/calico-tls" - if fileExists(getHostPath("/host/etc/cni/net.d/calico-tls/etcd-cert")) { + if fileExists("/host/etc/cni/net.d/calico-tls/etcd-cert") { etcdCertFile := fmt.Sprintf("%s/etcd-cert", hostSecretsDir) netconf = strings.Replace(netconf, "__ETCD_CERT_FILE__", etcdCertFile, -1) } else { netconf = strings.Replace(netconf, "__ETCD_CERT_FILE__", "", -1) } - if fileExists(getHostPath("/host/etc/cni/net.d/calico-tls/etcd-ca")) { + if fileExists("/host/etc/cni/net.d/calico-tls/etcd-ca") { etcdCACertFile := fmt.Sprintf("%s/etcd-ca", hostSecretsDir) netconf = strings.Replace(netconf, "__ETCD_CA_CERT_FILE__", etcdCACertFile, -1) } else { netconf = strings.Replace(netconf, "__ETCD_CA_CERT_FILE__", "", -1) } - if fileExists(getHostPath("/host/etc/cni/net.d/calico-tls/etcd-key")) { + if fileExists("/host/etc/cni/net.d/calico-tls/etcd-key") { etcdKeyFile := fmt.Sprintf("%s/etcd-key", hostSecretsDir) netconf = strings.Replace(netconf, "__ETCD_KEY_FILE__", etcdKeyFile, -1) } else { @@ -397,7 +388,7 @@ func writeCNIConfig(c config) { // Write out the file. name := getEnv("CNI_CONF_NAME", "10-calico.conflist") - path := getHostPath(fmt.Sprintf("/host/etc/cni/net.d/%s", name)) + path := fmt.Sprintf("/host/etc/cni/net.d/%s", name) err = os.WriteFile(path, []byte(netconf), 0644) if err != nil { logrus.Fatal(err) @@ -407,15 +398,15 @@ func writeCNIConfig(c config) { if err != nil { logrus.Fatal(err) } - logrus.Infof("Created %s", getHostPath(fmt.Sprintf("/host/etc/cni/net.d/%s", name))) + logrus.Infof("Created %s", fmt.Sprintf("/host/etc/cni/net.d/%s", name)) text := string(content) fmt.Println(text) // Remove any old config file, if one exists. oldName := getEnv("CNI_OLD_CONF_NAME", "10-calico.conflist") if name != oldName { - logrus.Infof("Removing %s", getHostPath(fmt.Sprintf("/host/etc/cni/net.d/%s", oldName))) - if err := os.Remove(getHostPath(fmt.Sprintf("/host/etc/cni/net.d/%s", oldName))); err != nil { + logrus.Infof("Removing %s", fmt.Sprintf("/host/etc/cni/net.d/%s", oldName)) + if err := os.Remove(fmt.Sprintf("/host/etc/cni/net.d/%s", oldName)); err != nil { logrus.WithError(err).Warnf("Failed to remove %s", oldName) } } @@ -506,7 +497,7 @@ current-context: calico-context` data = strings.Replace(data, "__TLS_CFG__", ca, -1) } - if err := os.WriteFile(getHostPath("/host/etc/cni/net.d/calico-kubeconfig"), []byte(data), 0600); err != nil { + if err := os.WriteFile("/host/etc/cni/net.d/calico-kubeconfig", []byte(data), 0600); err != nil { logrus.Fatal(err) } } From d28c91222930d3ec1a665188f6d274dc5280ab93 Mon Sep 17 00:00:00 2001 From: Pedro Coutinho Date: Wed, 19 Jul 2023 11:47:28 -0700 Subject: [PATCH 04/16] Remove kube-proxy service from host in unistall-calico-hpc.ps1 --- node/windows-packaging/CalicoWindows/uninstall-calico-hpc.ps1 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/node/windows-packaging/CalicoWindows/uninstall-calico-hpc.ps1 b/node/windows-packaging/CalicoWindows/uninstall-calico-hpc.ps1 index aad64d756b9..8223ecee796 100644 --- a/node/windows-packaging/CalicoWindows/uninstall-calico-hpc.ps1 +++ b/node/windows-packaging/CalicoWindows/uninstall-calico-hpc.ps1 @@ -76,5 +76,9 @@ Remove-CalicoService CalicoFelix Remove-CalicoService CalicoNode Remove-CalicoService CalicoUpgrade +Write-Host "Stopping and removing kube-proxy service if it is present..." +Write-Host "It is recommended to run kube-proxy as kubernetes daemonset instead" +Remove-CalicoService kube-proxy + Get-Module 'calico' | Remove-Module -Force Write-Host "Done." From 58c16954adcfd733be39d06f96147019dd1dd9d5 Mon Sep 17 00:00:00 2001 From: Pedro Coutinho Date: Tue, 25 Jul 2023 09:17:34 -0700 Subject: [PATCH 05/16] Add quotes to windows cni plugin template so that it's valid JSON even before substitution (this helps with operator testing). --- cni-plugin/pkg/install/install_windows.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cni-plugin/pkg/install/install_windows.go b/cni-plugin/pkg/install/install_windows.go index dbc4f493fdd..1f2002073f1 100644 --- a/cni-plugin/pkg/install/install_windows.go +++ b/cni-plugin/pkg/install/install_windows.go @@ -44,7 +44,7 @@ func defaultNetConf() string { }, "log_level": "__LOG_LEVEL__", "log_file_path": "__LOG_FILE_PATH__", - "windows_loopback_DSR": __DSR_SUPPORT__, + "windows_loopback_DSR": "__DSR_SUPPORT__", "capabilities": {"dns": true}, "DNS": { "Nameservers": [__KUBERNETES_DNS_SERVERS__], @@ -154,9 +154,9 @@ func replacePlatformSpecificVars(c config, netconf string) string { } supportsDSR := (winVerInt == 1809 && buildNumInt >= 17763 && halVerInt >= 1432) || (winVerInt >= 1903 && buildNumInt >= 18317) if supportsDSR { - netconf = strings.Replace(netconf, "__DSR_SUPPORT__", "true", -1) + netconf = strings.Replace(netconf, `"__DSR_SUPPORT__"`, "true", -1) } else { - netconf = strings.Replace(netconf, "__DSR_SUPPORT__", "false", -1) + netconf = strings.Replace(netconf, `"__DSR_SUPPORT__"`, "false", -1) } return netconf From e2029705bca0017f022177ae8c99d96ef150ca13 Mon Sep 17 00:00:00 2001 From: Pedro Coutinho Date: Thu, 27 Jul 2023 10:08:35 -0700 Subject: [PATCH 06/16] Fix cni-plugin Makefile for target image-windows --- cni-plugin/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cni-plugin/Makefile b/cni-plugin/Makefile index f034d5c076a..29b853c6549 100644 --- a/cni-plugin/Makefile +++ b/cni-plugin/Makefile @@ -303,7 +303,7 @@ image-windows: setup-windows-builder Dockerfile-windows build fetch-win-cni-bins image="$${registry}/$(CNI_PLUGIN_IMAGE_WINDOWS):$${image_version}"; \ docker buildx build \ --platform windows/amd64 \ - --output=type=image,push=$* \ + --output=type=image,push=$${push} \ -t $${image} \ --pull \ --no-cache \ From d177bbdbafc8e9ee6b1c92287eee9df088baf0fe Mon Sep 17 00:00:00 2001 From: Pedro Coutinho Date: Tue, 1 Aug 2023 17:48:23 -0700 Subject: [PATCH 07/16] image, build and file path changes to make windows HPC compatible with containerd v1.6+ (instead of requiring v1.7+). --- cni-plugin/Dockerfile-windows | 7 +- cni-plugin/Makefile | 84 ++++++++++++++----- cni-plugin/pkg/install/install.go | 46 ++++++---- cni-plugin/pkg/install/install_windows.go | 3 +- libcalico-go/lib/winutils/winutils.go | 8 ++ metadata.mk | 3 + node/Dockerfile-windows | 7 +- node/Makefile | 80 ++++++++++++++---- .../CalicoWindows/uninstall-calico-hpc.ps1 | 6 +- 9 files changed, 183 insertions(+), 61 deletions(-) diff --git a/cni-plugin/Dockerfile-windows b/cni-plugin/Dockerfile-windows index d366e1e5339..cd40b2e9641 100644 --- a/cni-plugin/Dockerfile-windows +++ b/cni-plugin/Dockerfile-windows @@ -12,9 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG WINDOWS_HPC_VERSION +# FIXME: Use WINDOWS_HPC_VERSION and image instead of nanoserver and WINDOWS_VERSIONS when containerd v1.6 is EOL'd +# ARG WINDOWS_HPC_VERSION +ARG WINDOWS_VERSION -FROM mcr.microsoft.com/oss/kubernetes/windows-host-process-containers-base-image:${WINDOWS_HPC_VERSION} +# FROM mcr.microsoft.com/oss/kubernetes/windows-host-process-containers-base-image:${WINDOWS_HPC_VERSION} +FROM mcr.microsoft.com/windows/nanoserver:${WINDOWS_VERSION} ARG GIT_VERSION=unknown diff --git a/cni-plugin/Makefile b/cni-plugin/Makefile index 29b853c6549..f870d01cda9 100644 --- a/cni-plugin/Makefile +++ b/cni-plugin/Makefile @@ -67,7 +67,7 @@ clean: clean-windows rm -rf dist/ clean-windows: clean-windows-builder - rm -rf $(WINDOWS_BIN) + rm -rf $(WINDOWS_BIN) $(WINDOWS_DIST) ############################################################################### # Building the binary @@ -293,29 +293,75 @@ release-publish-latest: release-prereqs $(MAKE) push-images-to-registries push-manifests RELEASE=true IMAGETAG=latest RELEASE=$(RELEASE) CONFIRM=$(CONFIRM) +# FIXME: Use WINDOWS_HPC_VERSION and image instead of nanoserver and WINDOWS_VERSIONS when containerd v1.6 is EOL'd +# .PHONY: image-windows release-windows +# # Build Windows image and possibly push it to $DEV_REGISTRIES +# image-windows: setup-windows-builder Dockerfile-windows build fetch-win-cni-bins +# push="$${PUSH:-false}"; \ +# image_version="$${VERSION:-latest}"; \ +# for registry in $(DEV_REGISTRIES); do \ +# echo Building and pushing Windows image to $${registry}; \ +# image="$${registry}/$(CNI_PLUGIN_IMAGE_WINDOWS):$${image_version}"; \ +# docker buildx build \ +# --platform windows/amd64 \ +# --output=type=image,push=$${push} \ +# -t $${image} \ +# --pull \ +# --no-cache \ +# --build-arg GIT_VERSION=$(GIT_VERSION) \ +# --build-arg WINDOWS_HPC_VERSION=$(WINDOWS_HPC_VERSION) \ +# -f Dockerfile-windows .; \ +# done ; + +# # Build and push Windows image +# release-windows: release-prereqs clean-windows +# $(MAKE) image-windows PUSH=true + +WINDOWS_DIST = dist/windows + +$(WINDOWS_DIST)/$(CNI_PLUGIN_IMAGE_WINDOWS)-$(GIT_VERSION)-%.tar: sub-image-windows-$* + +sub-image-windows-%: setup-windows-builder Dockerfile-windows build fetch-win-cni-bins + # ensure dir for windows image tars exits + -mkdir -p $(WINDOWS_DIST) + docker buildx build \ + --platform windows/amd64 \ + --output=type=docker,dest=$(CURDIR)/$(WINDOWS_DIST)/$(CNI_PLUGIN_IMAGE_WINDOWS)-$(GIT_VERSION)-$*.tar \ + --pull \ + -t $(CNI_PLUGIN_IMAGE_WINDOWS):latest \ + --build-arg GIT_VERSION=$(GIT_VERSION) \ + --build-arg=WINDOWS_VERSION=$* \ + -f Dockerfile-windows . + .PHONY: image-windows release-windows -# Build Windows image and possibly push it to $DEV_REGISTRIES -image-windows: setup-windows-builder Dockerfile-windows build fetch-win-cni-bins - push="$${PUSH:-false}"; \ +image-windows: setup-windows-builder + for version in $(WINDOWS_VERSIONS); do \ + $(MAKE) sub-image-windows-$${version}; \ + done; + +release-windows: clean-windows image-windows image_version="$${VERSION:-latest}"; \ for registry in $(DEV_REGISTRIES); do \ - echo Building and pushing Windows image to $${registry}; \ - image="$${registry}/$(CNI_PLUGIN_IMAGE_WINDOWS):$${image_version}"; \ - docker buildx build \ - --platform windows/amd64 \ - --output=type=image,push=$${push} \ - -t $${image} \ - --pull \ - --no-cache \ - --build-arg GIT_VERSION=$(GIT_VERSION) \ - --build-arg WINDOWS_HPC_VERSION=$(WINDOWS_HPC_VERSION) \ - -f Dockerfile-windows .; \ + echo Pushing Windows images to $${registry}; \ + all_images=""; \ + manifest_image="$${registry}/$(CNI_PLUGIN_IMAGE_WINDOWS):$${image_version}"; \ + for win_ver in $(WINDOWS_VERSIONS); do \ + image_tar="$(WINDOWS_DIST)/$(CNI_PLUGIN_IMAGE_WINDOWS)-$(GIT_VERSION)-$${win_ver}.tar"; \ + image="$${registry}/$(CNI_PLUGIN_IMAGE_WINDOWS):$${image_version}-windows-$${win_ver}"; \ + echo Pushing image $${image} ...; \ + $(CRANE_BINDMOUNT) push $${image_tar} $${image}$(double_quote) & \ + all_images="$${all_images} $${image}"; \ + done; \ + wait; \ + $(DOCKER_MANIFEST) create --amend $${manifest_image} $${all_images}; \ + for win_ver in $(WINDOWS_VERSIONS); do \ + version=$$(docker manifest inspect mcr.microsoft.com/windows/nanoserver:$${win_ver} | grep "os.version" | head -n 1 | awk -F\" '{print $$4}'); \ + image="$${registry}/$(CNI_PLUGIN_IMAGE_WINDOWS):$${image_version}-windows-$${win_ver}"; \ + $(DOCKER_MANIFEST) annotate --os windows --arch amd64 --os-version $${version} $${manifest_image} $${image}; \ + done; \ + $(DOCKER_MANIFEST) push --purge $${manifest_image}; \ done ; -# Build and push Windows image -release-windows: release-prereqs clean-windows - $(MAKE) image-windows PUSH=true - ############################################################################### # Developer helper scripts (not used by build or test) ############################################################################### diff --git a/cni-plugin/pkg/install/install.go b/cni-plugin/pkg/install/install.go index aac746da362..f4010a41063 100644 --- a/cni-plugin/pkg/install/install.go +++ b/cni-plugin/pkg/install/install.go @@ -22,6 +22,7 @@ import ( "io" "os" "os/exec" + "path/filepath" "runtime" "strings" "time" @@ -119,6 +120,21 @@ func loadConfig() config { return c } +// getHostPath returns the mount paths for a container +// In the case of Windows HostProcess containers this prepends the CONTAINER_SANDBOX_MOUNT_POINT env variable +// for other operating systems or if the sandbox env variable is not set it returns the standard mount points +// see https://kubernetes.io/docs/tasks/configure-pod-container/create-hostprocess-pod/#volume-mounts +// FIXME: this will no longer be needed when containerd v1.6 is EOL'd +func getHostPath(path string) string { + if runtime.GOOS == "windows" { + sandbox := os.Getenv("CONTAINER_SANDBOX_MOUNT_POINT") + // join them and return with forward slashes so it can be serialized properly in json later if required + path := filepath.Join(sandbox, path) + return filepath.ToSlash(path) + } + return path +} + func Install() error { // Make sure the RNG is seeded. seedrng.EnsureSeeded() @@ -127,7 +143,7 @@ func Install() error { logrus.SetFormatter(&logutils.Formatter{Component: "cni-installer"}) // Clean up any existing binaries / config / assets. - if err := os.RemoveAll("/host/etc/cni/net.d/calico-tls"); err != nil && !os.IsNotExist(err) { + if err := os.RemoveAll(getHostPath("/host/etc/cni/net.d/calico-tls")); err != nil && !os.IsNotExist(err) { logrus.WithError(err).Warnf("Error removing old TLS directory") } @@ -161,14 +177,14 @@ func Install() error { // First check if the dir exists and has anything in it. if directoryExists(c.TLSAssetsDir) { logrus.Info("Installing any TLS assets") - mkdir("/host/etc/cni/net.d/calico-tls") - if err := copyFileAndPermissions(fmt.Sprintf("%s/%s", c.TLSAssetsDir, "etcd-ca"), "/host/etc/cni/net.d/calico-tls/etcd-ca"); err != nil { + mkdir(getHostPath("/host/etc/cni/net.d/calico-tls")) + if err := copyFileAndPermissions(fmt.Sprintf("%s/%s", c.TLSAssetsDir, "etcd-ca"), getHostPath("/host/etc/cni/net.d/calico-tls/etcd-ca")); err != nil { logrus.Warnf("Missing etcd-ca") } - if err := copyFileAndPermissions(fmt.Sprintf("%s/%s", c.TLSAssetsDir, "etcd-cert"), "/host/etc/cni/net.d/calico-tls/etcd-cert"); err != nil { + if err := copyFileAndPermissions(fmt.Sprintf("%s/%s", c.TLSAssetsDir, "etcd-cert"), getHostPath("/host/etc/cni/net.d/calico-tls/etcd-cert")); err != nil { logrus.Warnf("Missing etcd-cert") } - if err := copyFileAndPermissions(fmt.Sprintf("%s/%s", c.TLSAssetsDir, "etcd-key"), "/host/etc/cni/net.d/calico-tls/etcd-key"); err != nil { + if err := copyFileAndPermissions(fmt.Sprintf("%s/%s", c.TLSAssetsDir, "etcd-key"), getHostPath("/host/etc/cni/net.d/calico-tls/etcd-key")); err != nil { logrus.Warnf("Missing etcd-key") } } @@ -189,7 +205,7 @@ func Install() error { } // Place the new binaries if the directory is writeable. - dirs := []string{"/host/opt/cni/bin", "/host/secondary-bin-dir"} + dirs := []string{getHostPath("/host/opt/cni/bin"), getHostPath("/host/secondary-bin-dir")} binsWritten := false for _, d := range dirs { if err := fileutil.IsDirWriteable(d); err != nil { @@ -283,7 +299,7 @@ func Install() error { logrus.Warn(err) } for _, f := range files { - if err = copyFileAndPermissions(c.TLSAssetsDir+"/"+f.Name(), "/host/etc/cni/net.d/calico-tls/"+f.Name()); err != nil { + if err = copyFileAndPermissions(getHostPath(c.TLSAssetsDir+"/"+f.Name()), getHostPath("/host/etc/cni/net.d/calico-tls/"+f.Name())); err != nil { logrus.Warn(err) continue } @@ -358,21 +374,21 @@ func writeCNIConfig(c config) { // Replace etcd datastore variables. hostSecretsDir := c.CNINetDir + "/calico-tls" - if fileExists("/host/etc/cni/net.d/calico-tls/etcd-cert") { + if fileExists(getHostPath("/host/etc/cni/net.d/calico-tls/etcd-cert")) { etcdCertFile := fmt.Sprintf("%s/etcd-cert", hostSecretsDir) netconf = strings.Replace(netconf, "__ETCD_CERT_FILE__", etcdCertFile, -1) } else { netconf = strings.Replace(netconf, "__ETCD_CERT_FILE__", "", -1) } - if fileExists("/host/etc/cni/net.d/calico-tls/etcd-ca") { + if fileExists(getHostPath("/host/etc/cni/net.d/calico-tls/etcd-ca")) { etcdCACertFile := fmt.Sprintf("%s/etcd-ca", hostSecretsDir) netconf = strings.Replace(netconf, "__ETCD_CA_CERT_FILE__", etcdCACertFile, -1) } else { netconf = strings.Replace(netconf, "__ETCD_CA_CERT_FILE__", "", -1) } - if fileExists("/host/etc/cni/net.d/calico-tls/etcd-key") { + if fileExists(getHostPath("/host/etc/cni/net.d/calico-tls/etcd-key")) { etcdKeyFile := fmt.Sprintf("%s/etcd-key", hostSecretsDir) netconf = strings.Replace(netconf, "__ETCD_KEY_FILE__", etcdKeyFile, -1) } else { @@ -388,7 +404,7 @@ func writeCNIConfig(c config) { // Write out the file. name := getEnv("CNI_CONF_NAME", "10-calico.conflist") - path := fmt.Sprintf("/host/etc/cni/net.d/%s", name) + path := getHostPath(fmt.Sprintf("/host/etc/cni/net.d/%s", name)) err = os.WriteFile(path, []byte(netconf), 0644) if err != nil { logrus.Fatal(err) @@ -398,15 +414,15 @@ func writeCNIConfig(c config) { if err != nil { logrus.Fatal(err) } - logrus.Infof("Created %s", fmt.Sprintf("/host/etc/cni/net.d/%s", name)) + logrus.Infof("Created %s", getHostPath(fmt.Sprintf("/host/etc/cni/net.d/%s", name))) text := string(content) fmt.Println(text) // Remove any old config file, if one exists. oldName := getEnv("CNI_OLD_CONF_NAME", "10-calico.conflist") if name != oldName { - logrus.Infof("Removing %s", fmt.Sprintf("/host/etc/cni/net.d/%s", oldName)) - if err := os.Remove(fmt.Sprintf("/host/etc/cni/net.d/%s", oldName)); err != nil { + logrus.Infof("Removing %s", getHostPath(fmt.Sprintf("/host/etc/cni/net.d/%s", oldName))) + if err := os.Remove(getHostPath(fmt.Sprintf("/host/etc/cni/net.d/%s", oldName))); err != nil { logrus.WithError(err).Warnf("Failed to remove %s", oldName) } } @@ -497,7 +513,7 @@ current-context: calico-context` data = strings.Replace(data, "__TLS_CFG__", ca, -1) } - if err := os.WriteFile("/host/etc/cni/net.d/calico-kubeconfig", []byte(data), 0600); err != nil { + if err := os.WriteFile(getHostPath("/host/etc/cni/net.d/calico-kubeconfig"), []byte(data), 0600); err != nil { logrus.Fatal(err) } } diff --git a/cni-plugin/pkg/install/install_windows.go b/cni-plugin/pkg/install/install_windows.go index 1f2002073f1..a7e09310075 100644 --- a/cni-plugin/pkg/install/install_windows.go +++ b/cni-plugin/pkg/install/install_windows.go @@ -21,9 +21,8 @@ import ( "strconv" "strings" - "github.com/sirupsen/logrus" - "github.com/projectcalico/calico/libcalico-go/lib/winutils" + "github.com/sirupsen/logrus" ) func defaultNetConf() string { diff --git a/libcalico-go/lib/winutils/winutils.go b/libcalico-go/lib/winutils/winutils.go index 1372304b48e..4ad0b04e8d8 100644 --- a/libcalico-go/lib/winutils/winutils.go +++ b/libcalico-go/lib/winutils/winutils.go @@ -15,10 +15,18 @@ package winutils import ( "bytes" + "os" "os/exec" ) func Powershell(args ...string) (string, string, error) { + // Add default powershell to PATH + path := os.Getenv("PATH") + err := os.Setenv("PATH", path+";C:/Windows/System32/WindowsPowerShell/v1.0/") + if err != nil { + return "", "", err + } + ps, err := exec.LookPath("powershell.exe") if err != nil { return "", "", err diff --git a/metadata.mk b/metadata.mk index d086bbc1c18..d21d8f4fe66 100644 --- a/metadata.mk +++ b/metadata.mk @@ -35,5 +35,8 @@ DEV_REGISTRIES ?= calico # This variable is unused. Registries for releases are defined in hack/release/pkg/builder/builder.go # RELEASE_REGISTRIES = quay.io/calico docker.io/calico gcr.io/projectcalico-org eu.gcr.io/projectcalico-org asia.gcr.io/projectcalico-org us.gcr.io/projectcalico-org +# FIXME: Use WINDOWS_HPC_VERSION and remove WINDOWS_VERSIONS when containerd v1.6 is EOL'd # The Windows HPC container version used as base for Calico Windows images WINDOWS_HPC_VERSION ?= v1.0.0 +# The Windows versions used as base for Calico Windows images +WINDOWS_VERSIONS ?= 1809 ltsc2022 diff --git a/node/Dockerfile-windows b/node/Dockerfile-windows index 1a7715a7d7e..e9876ba83da 100644 --- a/node/Dockerfile-windows +++ b/node/Dockerfile-windows @@ -12,9 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG WINDOWS_HPC_VERSION +# FIXME: Use WINDOWS_HPC_VERSION and image instead of nanoserver and WINDOWS_VERSIONS when containerd v1.6 is EOL'd +# ARG WINDOWS_HPC_VERSION +ARG WINDOWS_VERSION -FROM mcr.microsoft.com/oss/kubernetes/windows-host-process-containers-base-image:${WINDOWS_HPC_VERSION} +#FROM mcr.microsoft.com/oss/kubernetes/windows-host-process-containers-base-image:${WINDOWS_HPC_VERSION} +FROM mcr.microsoft.com/windows/nanoserver:${WINDOWS_VERSION} ARG GIT_VERSION=unknown diff --git a/node/Makefile b/node/Makefile index 89064feaa30..1c54c10b4c4 100644 --- a/node/Makefile +++ b/node/Makefile @@ -721,25 +721,69 @@ endif $(DOCKER_MANIFEST) push --purge $${manifest_image}; \ done ; +# FIXME: Use WINDOWS_HPC_VERSION and image instead of nanoserver and WINDOWS_VERSIONS when containerd v1.6 is EOL'd +# .PHONY: image-windows release-windows +# # Build Windows image and possibly push it to $DEV_REGISTRIES +# image-windows: setup-windows-builder Dockerfile-windows $(WINDOWS_BINARY) $(WINDOWS_ARCHIVE_ROOT)/libs/hns/hns.psm1 $(WINDOWS_ARCHIVE_ROOT)/libs/calico/calico.psm1 $(WINDOWS_ARCHIVE_ROOT)/config-hpc.ps1 $(WINDOWS_ARCHIVE_ROOT)/felix/felix-service.ps1 $(WINDOWS_ARCHIVE_ROOT)/node/node-service.ps1 $(WINDOWS_ARCHIVE_ROOT)/uninstall-calico-hpc.ps1 windows-packaging/nssm.exe +# push="$${PUSH:-false}"; \ +# image_version="$${VERSION:-latest}"; \ +# for registry in $(DEV_REGISTRIES); do \ +# echo Building and pushing Windows image to $${registry}; \ +# image="$${registry}/$(WINDOWS_IMAGE):$${image_version}"; \ +# docker buildx build \ +# --platform windows/amd64 \ +# --output=type=image,push=$${push} \ +# -t $${image} \ +# --pull \ +# --no-cache \ +# --build-arg GIT_VERSION=$(GIT_VERSION) \ +# --build-arg WINDOWS_HPC_VERSION=$(WINDOWS_HPC_VERSION) \ +# -f Dockerfile-windows .; \ +# done ; + +# # Build and push Windows image +# release-windows: release-prereqs clean-windows +# $(MAKE) image-windows PUSH=true + +$(WINDOWS_DIST)/$(WINDOWS_IMAGE)-$(GIT_VERSION)-%.tar: sub-image-windows-$* + +sub-image-windows-%: Dockerfile-windows $(WINDOWS_BINARY) $(WINDOWS_ARCHIVE_ROOT)/libs/hns/hns.psm1 $(WINDOWS_ARCHIVE_ROOT)/libs/calico/calico.psm1 $(WINDOWS_ARCHIVE_ROOT)/config-hpc.ps1 $(WINDOWS_ARCHIVE_ROOT)/felix/felix-service.ps1 $(WINDOWS_ARCHIVE_ROOT)/node/node-service.ps1 $(WINDOWS_ARCHIVE_ROOT)/uninstall-calico-hpc.ps1 windows-packaging/nssm.exe + # ensure dir for windows image tars exits + -mkdir -p $(WINDOWS_DIST) + docker buildx build \ + --platform windows/amd64 \ + --output=type=docker,dest=$(CURDIR)/$(WINDOWS_DIST)/$(WINDOWS_IMAGE)-$(GIT_VERSION)-$*.tar \ + --pull \ + -t $(WINDOWS_IMAGE):latest \ + --build-arg GIT_VERSION=$(GIT_VERSION) \ + --build-arg=WINDOWS_VERSION=$* \ + -f Dockerfile-windows . + .PHONY: image-windows release-windows -# Build Windows image and possibly push it to $DEV_REGISTRIES -image-windows: setup-windows-builder Dockerfile-windows $(WINDOWS_BINARY) $(WINDOWS_ARCHIVE_ROOT)/libs/hns/hns.psm1 $(WINDOWS_ARCHIVE_ROOT)/libs/calico/calico.psm1 $(WINDOWS_ARCHIVE_ROOT)/config-hpc.ps1 $(WINDOWS_ARCHIVE_ROOT)/felix/felix-service.ps1 $(WINDOWS_ARCHIVE_ROOT)/node/node-service.ps1 $(WINDOWS_ARCHIVE_ROOT)/uninstall-calico-hpc.ps1 windows-packaging/nssm.exe - push="$${PUSH:-false}"; \ +image-windows: setup-windows-builder + for version in $(WINDOWS_VERSIONS); do \ + $(MAKE) sub-image-windows-$${version}; \ + done; + +release-windows: clean-windows image-windows image_version="$${VERSION:-latest}"; \ for registry in $(DEV_REGISTRIES); do \ - echo Building and pushing Windows image to $${registry}; \ - image="$${registry}/$(WINDOWS_IMAGE):$${image_version}"; \ - docker buildx build \ - --platform windows/amd64 \ - --output=type=image,push=$${push} \ - -t $${image} \ - --pull \ - --no-cache \ - --build-arg GIT_VERSION=$(GIT_VERSION) \ - --build-arg WINDOWS_HPC_VERSION=$(WINDOWS_HPC_VERSION) \ - -f Dockerfile-windows .; \ + echo Pushing Windows images to $${registry}; \ + all_images=""; \ + manifest_image="$${registry}/$(WINDOWS_IMAGE):$${image_version}"; \ + for win_ver in $(WINDOWS_VERSIONS); do \ + image_tar="$(WINDOWS_DIST)/$(WINDOWS_IMAGE)-$(GIT_VERSION)-$${win_ver}.tar"; \ + image="$${registry}/$(WINDOWS_IMAGE):$${image_version}-windows-$${win_ver}"; \ + echo Pushing image $${image} ...; \ + $(CRANE_BINDMOUNT) push $${image_tar} $${image}$(double_quote) & \ + all_images="$${all_images} $${image}"; \ + done; \ + wait; \ + $(DOCKER_MANIFEST) create --amend $${manifest_image} $${all_images}; \ + for win_ver in $(WINDOWS_VERSIONS); do \ + version=$$(docker manifest inspect mcr.microsoft.com/windows/nanoserver:$${win_ver} | grep "os.version" | head -n 1 | awk -F\" '{print $$4}'); \ + image="$${registry}/$(WINDOWS_IMAGE):$${image_version}-windows-$${win_ver}"; \ + $(DOCKER_MANIFEST) annotate --os windows --arch amd64 --os-version $${version} $${manifest_image} $${image}; \ + done; \ + $(DOCKER_MANIFEST) push --purge $${manifest_image}; \ done ; - -# Build and push Windows image -release-windows: release-prereqs clean-windows - $(MAKE) image-windows PUSH=true diff --git a/node/windows-packaging/CalicoWindows/uninstall-calico-hpc.ps1 b/node/windows-packaging/CalicoWindows/uninstall-calico-hpc.ps1 index 8223ecee796..55c86357dff 100644 --- a/node/windows-packaging/CalicoWindows/uninstall-calico-hpc.ps1 +++ b/node/windows-packaging/CalicoWindows/uninstall-calico-hpc.ps1 @@ -38,7 +38,7 @@ function Remove-CalicoService($ServiceName) } } -if (-not (Test-Path "env:$CNI_NET_DIR")) +if (-not (Test-Path "$env:CNI_NET_DIR")) { Write-Host "CNI_NET_DIR env var not set, skipping Calico CNI config cleanup" } else @@ -49,7 +49,7 @@ if (-not (Test-Path "env:$CNI_NET_DIR")) rm $cniConfFile } - if (Test-Path "env:$CNI_CONF_NAME") { + if (Test-Path "$env:CNI_CONF_NAME") { $cniConfListFile = "$env:CONTAINER_SANDBOX_MOUNT_POINT/host/$env:CNI_NET_DIR/$env:CNI_CONF_NAME" if (Test-Path $cniConfListFile) { Write-Host "Removing Calico CNI conf file at $cniConfListFile ..." @@ -58,7 +58,7 @@ if (-not (Test-Path "env:$CNI_NET_DIR")) } } -if (-not (Test-Path "env:$CNI_BIN_DIR")) +if (-not (Test-Path "$env:CNI_BIN_DIR")) { Write-Host "CNI_BIN_DIR env var not set, skipping Calico CNI binary cleanup" } else From c81279d57584d5756cac771d76e78ce666052082 Mon Sep 17 00:00:00 2001 From: Pedro Coutinho Date: Thu, 3 Aug 2023 16:32:32 -0700 Subject: [PATCH 08/16] Make HPC work with containerd v1.6 (by referring to all paths prefixed with '$env:CONTAINER_SANDBOX_MOUNT_POINT'). Fix logic regarding CNI paths in uninstall-calico-hpc.ps1. --- cni-plugin/pkg/install/install.go | 91 ++++++++++++------- cni-plugin/pkg/install/install_windows.go | 8 +- libcalico-go/lib/winutils/winutils.go | 20 ++++ node/pkg/cni/token_watch.go | 13 +-- .../CalicoWindows/uninstall-calico-hpc.ps1 | 17 +++- 5 files changed, 102 insertions(+), 47 deletions(-) diff --git a/cni-plugin/pkg/install/install.go b/cni-plugin/pkg/install/install.go index f4010a41063..8cffad330d5 100644 --- a/cni-plugin/pkg/install/install.go +++ b/cni-plugin/pkg/install/install.go @@ -20,9 +20,9 @@ import ( "encoding/json" "fmt" "io" + "net" "os" "os/exec" - "path/filepath" "runtime" "strings" "time" @@ -34,10 +34,12 @@ import ( "go.etcd.io/etcd/client/pkg/v3/fileutil" "k8s.io/client-go/rest" + certutil "k8s.io/client-go/util/cert" "github.com/projectcalico/calico/libcalico-go/lib/logutils" "github.com/projectcalico/calico/libcalico-go/lib/names" "github.com/projectcalico/calico/libcalico-go/lib/seedrng" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" "github.com/projectcalico/calico/node/pkg/cni" ) @@ -120,21 +122,6 @@ func loadConfig() config { return c } -// getHostPath returns the mount paths for a container -// In the case of Windows HostProcess containers this prepends the CONTAINER_SANDBOX_MOUNT_POINT env variable -// for other operating systems or if the sandbox env variable is not set it returns the standard mount points -// see https://kubernetes.io/docs/tasks/configure-pod-container/create-hostprocess-pod/#volume-mounts -// FIXME: this will no longer be needed when containerd v1.6 is EOL'd -func getHostPath(path string) string { - if runtime.GOOS == "windows" { - sandbox := os.Getenv("CONTAINER_SANDBOX_MOUNT_POINT") - // join them and return with forward slashes so it can be serialized properly in json later if required - path := filepath.Join(sandbox, path) - return filepath.ToSlash(path) - } - return path -} - func Install() error { // Make sure the RNG is seeded. seedrng.EnsureSeeded() @@ -143,7 +130,7 @@ func Install() error { logrus.SetFormatter(&logutils.Formatter{Component: "cni-installer"}) // Clean up any existing binaries / config / assets. - if err := os.RemoveAll(getHostPath("/host/etc/cni/net.d/calico-tls")); err != nil && !os.IsNotExist(err) { + if err := os.RemoveAll(winutils.GetHostPath("/host/etc/cni/net.d/calico-tls")); err != nil && !os.IsNotExist(err) { logrus.WithError(err).Warnf("Error removing old TLS directory") } @@ -153,12 +140,13 @@ func Install() error { // Determine if we're running as a Kubernetes pod. var kubecfg *rest.Config - serviceAccountTokenFile := "/var/run/secrets/kubernetes.io/serviceaccount/token" + serviceAccountTokenFile := winutils.GetHostPath("/var/run/secrets/kubernetes.io/serviceaccount/token") c.ServiceAccountToken = make([]byte, 0) var err error if fileExists(serviceAccountTokenFile) { logrus.Info("Running as a Kubernetes pod") - kubecfg, err = rest.InClusterConfig() + // FIXME: get rid of this and call rest.InClusterConfig() directly when containerd v1.6 is EOL'd + kubecfg, err = getInClusterConfig() if err != nil { return err } @@ -177,14 +165,14 @@ func Install() error { // First check if the dir exists and has anything in it. if directoryExists(c.TLSAssetsDir) { logrus.Info("Installing any TLS assets") - mkdir(getHostPath("/host/etc/cni/net.d/calico-tls")) - if err := copyFileAndPermissions(fmt.Sprintf("%s/%s", c.TLSAssetsDir, "etcd-ca"), getHostPath("/host/etc/cni/net.d/calico-tls/etcd-ca")); err != nil { + mkdir(winutils.GetHostPath("/host/etc/cni/net.d/calico-tls")) + if err := copyFileAndPermissions(fmt.Sprintf("%s/%s", c.TLSAssetsDir, "etcd-ca"), winutils.GetHostPath("/host/etc/cni/net.d/calico-tls/etcd-ca")); err != nil { logrus.Warnf("Missing etcd-ca") } - if err := copyFileAndPermissions(fmt.Sprintf("%s/%s", c.TLSAssetsDir, "etcd-cert"), getHostPath("/host/etc/cni/net.d/calico-tls/etcd-cert")); err != nil { + if err := copyFileAndPermissions(fmt.Sprintf("%s/%s", c.TLSAssetsDir, "etcd-cert"), winutils.GetHostPath("/host/etc/cni/net.d/calico-tls/etcd-cert")); err != nil { logrus.Warnf("Missing etcd-cert") } - if err := copyFileAndPermissions(fmt.Sprintf("%s/%s", c.TLSAssetsDir, "etcd-key"), getHostPath("/host/etc/cni/net.d/calico-tls/etcd-key")); err != nil { + if err := copyFileAndPermissions(fmt.Sprintf("%s/%s", c.TLSAssetsDir, "etcd-key"), winutils.GetHostPath("/host/etc/cni/net.d/calico-tls/etcd-key")); err != nil { logrus.Warnf("Missing etcd-key") } } @@ -205,7 +193,7 @@ func Install() error { } // Place the new binaries if the directory is writeable. - dirs := []string{getHostPath("/host/opt/cni/bin"), getHostPath("/host/secondary-bin-dir")} + dirs := []string{winutils.GetHostPath("/host/opt/cni/bin"), winutils.GetHostPath("/host/secondary-bin-dir")} binsWritten := false for _, d := range dirs { if err := fileutil.IsDirWriteable(d); err != nil { @@ -299,7 +287,7 @@ func Install() error { logrus.Warn(err) } for _, f := range files { - if err = copyFileAndPermissions(getHostPath(c.TLSAssetsDir+"/"+f.Name()), getHostPath("/host/etc/cni/net.d/calico-tls/"+f.Name())); err != nil { + if err = copyFileAndPermissions(winutils.GetHostPath(c.TLSAssetsDir+"/"+f.Name()), winutils.GetHostPath("/host/etc/cni/net.d/calico-tls/"+f.Name())); err != nil { logrus.Warn(err) continue } @@ -374,21 +362,21 @@ func writeCNIConfig(c config) { // Replace etcd datastore variables. hostSecretsDir := c.CNINetDir + "/calico-tls" - if fileExists(getHostPath("/host/etc/cni/net.d/calico-tls/etcd-cert")) { + if fileExists(winutils.GetHostPath("/host/etc/cni/net.d/calico-tls/etcd-cert")) { etcdCertFile := fmt.Sprintf("%s/etcd-cert", hostSecretsDir) netconf = strings.Replace(netconf, "__ETCD_CERT_FILE__", etcdCertFile, -1) } else { netconf = strings.Replace(netconf, "__ETCD_CERT_FILE__", "", -1) } - if fileExists(getHostPath("/host/etc/cni/net.d/calico-tls/etcd-ca")) { + if fileExists(winutils.GetHostPath("/host/etc/cni/net.d/calico-tls/etcd-ca")) { etcdCACertFile := fmt.Sprintf("%s/etcd-ca", hostSecretsDir) netconf = strings.Replace(netconf, "__ETCD_CA_CERT_FILE__", etcdCACertFile, -1) } else { netconf = strings.Replace(netconf, "__ETCD_CA_CERT_FILE__", "", -1) } - if fileExists(getHostPath("/host/etc/cni/net.d/calico-tls/etcd-key")) { + if fileExists(winutils.GetHostPath("/host/etc/cni/net.d/calico-tls/etcd-key")) { etcdKeyFile := fmt.Sprintf("%s/etcd-key", hostSecretsDir) netconf = strings.Replace(netconf, "__ETCD_KEY_FILE__", etcdKeyFile, -1) } else { @@ -404,7 +392,7 @@ func writeCNIConfig(c config) { // Write out the file. name := getEnv("CNI_CONF_NAME", "10-calico.conflist") - path := getHostPath(fmt.Sprintf("/host/etc/cni/net.d/%s", name)) + path := winutils.GetHostPath(fmt.Sprintf("/host/etc/cni/net.d/%s", name)) err = os.WriteFile(path, []byte(netconf), 0644) if err != nil { logrus.Fatal(err) @@ -414,15 +402,15 @@ func writeCNIConfig(c config) { if err != nil { logrus.Fatal(err) } - logrus.Infof("Created %s", getHostPath(fmt.Sprintf("/host/etc/cni/net.d/%s", name))) + logrus.Infof("Created %s", winutils.GetHostPath(fmt.Sprintf("/host/etc/cni/net.d/%s", name))) text := string(content) fmt.Println(text) // Remove any old config file, if one exists. oldName := getEnv("CNI_OLD_CONF_NAME", "10-calico.conflist") if name != oldName { - logrus.Infof("Removing %s", getHostPath(fmt.Sprintf("/host/etc/cni/net.d/%s", oldName))) - if err := os.Remove(getHostPath(fmt.Sprintf("/host/etc/cni/net.d/%s", oldName))); err != nil { + logrus.Infof("Removing %s", winutils.GetHostPath(fmt.Sprintf("/host/etc/cni/net.d/%s", oldName))) + if err := os.Remove(winutils.GetHostPath(fmt.Sprintf("/host/etc/cni/net.d/%s", oldName))); err != nil { logrus.WithError(err).Warnf("Failed to remove %s", oldName) } } @@ -513,7 +501,7 @@ current-context: calico-context` data = strings.Replace(data, "__TLS_CFG__", ca, -1) } - if err := os.WriteFile(getHostPath("/host/etc/cni/net.d/calico-kubeconfig"), []byte(data), 0600); err != nil { + if err := os.WriteFile(winutils.GetHostPath("/host/etc/cni/net.d/calico-kubeconfig"), []byte(data), 0600); err != nil { logrus.Fatal(err) } } @@ -623,3 +611,40 @@ func destinationUptoDate(src, dst string) (bool, error) { // slice of bytes. } } + +// FIXME: get rid of this and call rest.InClusterConfig() directly when containerd v1.6 is EOL'd +// getInClusterConfig returns a config object which uses the service account +// kubernetes gives to pods. It's intended for clients that expect to be +// running inside a pod running on kubernetes. It will return ErrNotInCluster +// if called from a process not running in a kubernetes environment. +// It is a copy of InClusterConfig() from k8s.io/client-go/rest but using +// winutils.GetHostPath() for the file paths, so that Windows hostprocess +// containers on containerd v1.6 can work with the in-cluster config. +func getInClusterConfig() (*rest.Config, error) { + tokenFile := winutils.GetHostPath("/var/run/secrets/kubernetes.io/serviceaccount/token") + rootCAFile := winutils.GetHostPath("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt") + host, port := os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT") + if len(host) == 0 || len(port) == 0 { + return nil, rest.ErrNotInCluster + } + + token, err := os.ReadFile(tokenFile) + if err != nil { + return nil, err + } + + tlsClientConfig := rest.TLSClientConfig{} + + if _, err := certutil.NewPool(rootCAFile); err != nil { + logrus.Errorf("Expected to load root CA config from %s, but got err: %v", rootCAFile, err) + } else { + tlsClientConfig.CAFile = rootCAFile + } + + return &rest.Config{ + Host: "https://" + net.JoinHostPort(host, port), + TLSClientConfig: tlsClientConfig, + BearerToken: string(token), + BearerTokenFile: tokenFile, + }, nil +} diff --git a/cni-plugin/pkg/install/install_windows.go b/cni-plugin/pkg/install/install_windows.go index a7e09310075..928a3bd99da 100644 --- a/cni-plugin/pkg/install/install_windows.go +++ b/cni-plugin/pkg/install/install_windows.go @@ -18,6 +18,7 @@ package install import ( "fmt" + "path/filepath" "strconv" "strings" @@ -27,7 +28,7 @@ import ( func defaultNetConf() string { netconf := `{ - "name": "k8s-pod-network", + "name": "Calico", "cniVersion": "0.3.1", "plugins": [ { @@ -96,8 +97,8 @@ func defaultNetConf() string { // Perform replacement of windows variables func replacePlatformSpecificVars(c config, netconf string) string { - - kubeconfigPath := "c:/" + c.CNINetDir + "/calico-kubeconfig" + kubeconfigPath := filepath.Join("c:", strings.TrimLeft(c.CNINetDir, "c:"), "/calico-kubeconfig") + kubeconfigPath = filepath.ToSlash(kubeconfigPath) netconf = strings.Replace(netconf, "__KUBECONFIG_FILEPATH__", kubeconfigPath, -1) netconf = strings.Replace(netconf, "__LOG_FILE_PATH__", getEnv("LOG_FILE_PATH", "c:/var/log/calico/cni/cni.log"), -1) @@ -152,6 +153,7 @@ func replacePlatformSpecificVars(c config, netconf string) string { logrus.Fatalf("Error converting halVer to int\nerror: %s", err) } supportsDSR := (winVerInt == 1809 && buildNumInt >= 17763 && halVerInt >= 1432) || (winVerInt >= 1903 && buildNumInt >= 18317) + // Remove the quotes when replacing with boolean values (the quotes are in so that the template if valid JSON even before replacing) if supportsDSR { netconf = strings.Replace(netconf, `"__DSR_SUPPORT__"`, "true", -1) } else { diff --git a/libcalico-go/lib/winutils/winutils.go b/libcalico-go/lib/winutils/winutils.go index 4ad0b04e8d8..fc6b0f1f89b 100644 --- a/libcalico-go/lib/winutils/winutils.go +++ b/libcalico-go/lib/winutils/winutils.go @@ -17,6 +17,9 @@ import ( "bytes" "os" "os/exec" + "path/filepath" + "runtime" + "strings" ) func Powershell(args ...string) (string, string, error) { @@ -47,3 +50,20 @@ func Powershell(args ...string) (string, string, error) { return stdout.String(), stderr.String(), err } + +// GetHostPath returns the mount paths for a container +// In the case of Windows HostProcess containers this prepends the CONTAINER_SANDBOX_MOUNT_POINT env variable +// for other operating systems or if the sandbox env variable is not set it returns the standard mount points +// see https://kubernetes.io/docs/tasks/configure-pod-container/create-hostprocess-pod/#volume-mounts +// FIXME: this will no longer be needed when containerd v1.6 is EOL'd +func GetHostPath(path string) string { + if runtime.GOOS == "windows" { + sandbox := os.Getenv("CONTAINER_SANDBOX_MOUNT_POINT") + // join them and return with forward slashes so it can be serialized properly in json later if required + path := strings.TrimLeft(path, "c:") + path = strings.TrimLeft(path, "C:") + path = filepath.Join(sandbox, path, "c:") + return filepath.ToSlash(path) + } + return path +} diff --git a/node/pkg/cni/token_watch.go b/node/pkg/cni/token_watch.go index d6e5e6bdcaa..de5ebb6e1a9 100644 --- a/node/pkg/cni/token_watch.go +++ b/node/pkg/cni/token_watch.go @@ -12,6 +12,7 @@ import ( "sync" "time" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" "github.com/sirupsen/logrus" authenticationv1 "k8s.io/api/authentication/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -29,10 +30,9 @@ const ( defaultCNITokenValiditySeconds = 24 * 60 * 60 minTokenRetryDuration = 5 * time.Second defaultRefreshFraction = 4 + kubeconfigPath = "/host/etc/cni/net.d/calico-kubeconfig" ) -var kubeconfigPath string = "/host/etc/cni/net.d/calico-kubeconfig" - type TokenRefresher struct { tokenSupported bool tokenOnce *sync.Once @@ -56,7 +56,7 @@ type TokenUpdate struct { } func NamespaceOfUsedServiceAccount() string { - namespace, err := os.ReadFile(serviceAccountNamespace) + namespace, err := os.ReadFile(winutils.GetHostPath(serviceAccountNamespace)) if err != nil { logrus.WithError(err).Fatal("Failed to read service account namespace file") } @@ -179,7 +179,7 @@ func (t *TokenRefresher) tokenRequestSupported(clientset *kubernetes.Clientset) } func tokenUpdateFromFile() (TokenUpdate, error) { - tokenBytes, err := os.ReadFile(tokenFile) + tokenBytes, err := os.ReadFile(winutils.GetHostPath(tokenFile)) if err != nil { logrus.WithError(err).Error("Failed to read service account token file") return TokenUpdate{}, err @@ -276,9 +276,10 @@ current-context: calico-context` data := fmt.Sprintf(template, cfg.Host, base64.StdEncoding.EncodeToString(cfg.CAData), token) // Write the filled out config to disk. - if err := os.WriteFile(kubeconfigPath, []byte(data), 0600); err != nil { + if err := os.WriteFile(winutils.GetHostPath(kubeconfigPath), []byte(data), 0600); err != nil { logrus.WithError(err).Error("Failed to write CNI plugin kubeconfig file") return } - logrus.WithField("path", kubeconfigPath).Info("Wrote updated CNI kubeconfig file.") + // logrus.WithField("path", kubeconfigPath).Info("Wrote updated CNI kubeconfig file.") + logrus.WithField("path", winutils.GetHostPath(kubeconfigPath)).Info("Wrote updated CNI kubeconfig file.") } diff --git a/node/windows-packaging/CalicoWindows/uninstall-calico-hpc.ps1 b/node/windows-packaging/CalicoWindows/uninstall-calico-hpc.ps1 index 55c86357dff..219152fb6b8 100644 --- a/node/windows-packaging/CalicoWindows/uninstall-calico-hpc.ps1 +++ b/node/windows-packaging/CalicoWindows/uninstall-calico-hpc.ps1 @@ -38,19 +38,22 @@ function Remove-CalicoService($ServiceName) } } -if (-not (Test-Path "$env:CNI_NET_DIR")) +if ("$env:CNI_NET_DIR" -eq $null) { Write-Host "CNI_NET_DIR env var not set, skipping Calico CNI config cleanup" +} elseif (-not (Test-Path "$env:CONTAINER_SANDBOX_MOUNT_POINT/$env:CNI_NET_DIR")) +{ + Write-Host "$env:CNI_NET_DIR dir does not exist, skipping Calico CNI config cleanup" } else { - $cniConfFile = "$env:CONTAINER_SANDBOX_MOUNT_POINT/host/$env:CNI_NET_DIR/10-calico.conf" + $cniConfFile = "$env:CONTAINER_SANDBOX_MOUNT_POINT/$env:CNI_NET_DIR/10-calico.conf" if (Test-Path $cniConfFile) { Write-Host "Removing Calico CNI conf file at $cniConfFile ..." rm $cniConfFile } if (Test-Path "$env:CNI_CONF_NAME") { - $cniConfListFile = "$env:CONTAINER_SANDBOX_MOUNT_POINT/host/$env:CNI_NET_DIR/$env:CNI_CONF_NAME" + $cniConfListFile = "$env:CONTAINER_SANDBOX_MOUNT_POINT/$env:CNI_NET_DIR/$env:CNI_CONF_NAME" if (Test-Path $cniConfListFile) { Write-Host "Removing Calico CNI conf file at $cniConfListFile ..." rm $cniConfListFile @@ -58,12 +61,16 @@ if (-not (Test-Path "$env:CNI_NET_DIR")) } } -if (-not (Test-Path "$env:CNI_BIN_DIR")) + +if ("$env:CNI_BIN_DIR" -eq $null) { Write-Host "CNI_BIN_DIR env var not set, skipping Calico CNI binary cleanup" +} elseif (-not (Test-Path "$env:CONTAINER_SANDBOX_MOUNT_POINT/$env:CNI_BIN_DIR")) +{ + Write-Host "$env:CNI_BIN_DIR dir does not exist, skipping Calico CNI binary cleanup" } else { - $cniBinPath = "$env:CONTAINER_SANDBOX_MOUNT_POINT/host/$env:CNI_BIN_DIR/calico*.exe" + $cniBinPath = "$env:CONTAINER_SANDBOX_MOUNT_POINT/$env:CNI_BIN_DIR/calico*.exe" if (Test-Path $cniBinPath) { Write-Host "Removing Calico CNI binaries at $cniBinPath ..." rm $cniBinPath From bac65a572c6f1f425bd9d8b82fd3f76ef6e140aa Mon Sep 17 00:00:00 2001 From: Pedro Coutinho Date: Mon, 21 Aug 2023 14:19:05 -0700 Subject: [PATCH 09/16] Add support for multiple kubernetes service CIDRs Fix bug in GetHostPath() --- cni-plugin/pkg/install/install_windows.go | 43 ++++++++++++++++------- libcalico-go/lib/winutils/winutils.go | 2 +- 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/cni-plugin/pkg/install/install_windows.go b/cni-plugin/pkg/install/install_windows.go index 928a3bd99da..8a3ac0ca54c 100644 --- a/cni-plugin/pkg/install/install_windows.go +++ b/cni-plugin/pkg/install/install_windows.go @@ -75,19 +75,10 @@ func defaultNetConf() string { "Name": "EndpointPolicy", "Value": { "Type": "OutBoundNAT", - "ExceptionList": [ - "__KUBERNETES_SERVICE_CIDR__" - ] + "ExceptionList": [__KUBERNETES_SERVICE_CIDRS__] } }, - { - "Name": "EndpointPolicy", - "Value": { - "Type": "__ROUTE_TYPE__", - "DestinationPrefix": "__KUBERNETES_SERVICE_CIDR__", - "NeedEncap": true - } - } +__KUBERNETES_ROUTE_POLICIES__ ] } ] @@ -102,8 +93,34 @@ func replacePlatformSpecificVars(c config, netconf string) string { netconf = strings.Replace(netconf, "__KUBECONFIG_FILEPATH__", kubeconfigPath, -1) netconf = strings.Replace(netconf, "__LOG_FILE_PATH__", getEnv("LOG_FILE_PATH", "c:/var/log/calico/cni/cni.log"), -1) + + // Support multiple KUBERNETES_SERVICE_CIDRS + serviceCIDRIPs := getEnv("KUBERNETES_SERVICE_CIDRS", "10.96.0.10") + serviceCIDRIPList := []string{} + for _, ip := range strings.Split(serviceCIDRIPs, ",") { + serviceCIDRIPList = append(serviceCIDRIPList, fmt.Sprintf("\"%s\"", strings.TrimSpace(ip))) + } + quotedServiceCIDRIPs := strings.Join(serviceCIDRIPList, ",") + netconf = strings.Replace(netconf, "__KUBERNETES_SERVICE_CIDRS__", quotedServiceCIDRIPs, -1) + + routePolicyList := []string{} + for _, ip := range serviceCIDRIPList { + routePolicy := fmt.Sprintf(` { + "Name": "EndpointPolicy", + "Value": { + "Type": "__ROUTE_TYPE__", + "DestinationPrefix": %s, + "NeedEncap": true + } + }`, ip) + routePolicyList = append(routePolicyList, routePolicy) + } + routePolicyListStr := strings.Join(routePolicyList, ",\n") + netconf = strings.Replace(netconf, "__KUBERNETES_ROUTE_POLICIES__", routePolicyListStr, -1) + + // __ROUTE_TYPE__ substitution must be done after __KUBERNETES_ROUTE_POLICIES__ because the latter contains the former. netconf = strings.Replace(netconf, "__ROUTE_TYPE__", getEnv("ROUTE_TYPE", "SDNROUTE"), -1) - netconf = strings.Replace(netconf, "__KUBERNETES_SERVICE_CIDR__", getEnv("KUBERNETES_SERVICE_CIDR", "10.96.0.0/12"), -1) + netconf = strings.Replace(netconf, "__VNI__", getEnv("VXLAN_VNI", "4096"), -1) netconf = strings.Replace(netconf, "__MAC_PREFIX__", getEnv("MAC_PREFIX", "0E-2A"), -1) @@ -153,7 +170,7 @@ func replacePlatformSpecificVars(c config, netconf string) string { logrus.Fatalf("Error converting halVer to int\nerror: %s", err) } supportsDSR := (winVerInt == 1809 && buildNumInt >= 17763 && halVerInt >= 1432) || (winVerInt >= 1903 && buildNumInt >= 18317) - // Remove the quotes when replacing with boolean values (the quotes are in so that the template if valid JSON even before replacing) + // Remove the quotes when replacing with boolean values (the quotes are in so that the template is valid JSON even before replacing) if supportsDSR { netconf = strings.Replace(netconf, `"__DSR_SUPPORT__"`, "true", -1) } else { diff --git a/libcalico-go/lib/winutils/winutils.go b/libcalico-go/lib/winutils/winutils.go index fc6b0f1f89b..bdf9546ea17 100644 --- a/libcalico-go/lib/winutils/winutils.go +++ b/libcalico-go/lib/winutils/winutils.go @@ -62,7 +62,7 @@ func GetHostPath(path string) string { // join them and return with forward slashes so it can be serialized properly in json later if required path := strings.TrimLeft(path, "c:") path = strings.TrimLeft(path, "C:") - path = filepath.Join(sandbox, path, "c:") + path = filepath.Join(sandbox, path) return filepath.ToSlash(path) } return path From 678854ed13f98239e01bacfcf40b8056636e6488 Mon Sep 17 00:00:00 2001 From: Pedro Coutinho Date: Wed, 23 Aug 2023 14:06:30 -0700 Subject: [PATCH 10/16] Run 'make fix' to fix 'make check-fmt' in cni-plugin and node --- cni-plugin/pkg/install/install_windows.go | 3 ++- node/pkg/cni/token_watch.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cni-plugin/pkg/install/install_windows.go b/cni-plugin/pkg/install/install_windows.go index 8a3ac0ca54c..89be5ca4a72 100644 --- a/cni-plugin/pkg/install/install_windows.go +++ b/cni-plugin/pkg/install/install_windows.go @@ -22,8 +22,9 @@ import ( "strconv" "strings" - "github.com/projectcalico/calico/libcalico-go/lib/winutils" "github.com/sirupsen/logrus" + + "github.com/projectcalico/calico/libcalico-go/lib/winutils" ) func defaultNetConf() string { diff --git a/node/pkg/cni/token_watch.go b/node/pkg/cni/token_watch.go index de5ebb6e1a9..0c58732f644 100644 --- a/node/pkg/cni/token_watch.go +++ b/node/pkg/cni/token_watch.go @@ -12,7 +12,6 @@ import ( "sync" "time" - "github.com/projectcalico/calico/libcalico-go/lib/winutils" "github.com/sirupsen/logrus" authenticationv1 "k8s.io/api/authentication/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -20,6 +19,8 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + + "github.com/projectcalico/calico/libcalico-go/lib/winutils" ) const ( From aad494bf5f5dab8d407a81d1d9ef8259f8040c07 Mon Sep 17 00:00:00 2001 From: Pedro Coutinho Date: Wed, 30 Aug 2023 12:54:24 -0700 Subject: [PATCH 11/16] Add InHostProcessContainer(), GetInClusterConfig(), BuildConfigFromFlags() to winutils.go and use them instead of upstream's InClusterConfig(), BuildConfigFromFlags() in order to get Windows HPC to work with containerd v1.6. (tests intentionally left untouched in order to try to catch discrepancies in behavior from upstream if/when they happen). Fix PATH in Windows dockerfiles. --- apiserver/cmd/apiserver/server/watch.go | 7 +- cni-plugin/Dockerfile-windows | 2 +- cni-plugin/pkg/install/install.go | 41 +--------- cni-plugin/pkg/k8s/k8s.go | 17 +++- cni-plugin/pkg/plugin/plugin.go | 4 +- confd/pkg/backends/calico/routes.go | 7 +- confd/pkg/backends/calico/secret_watcher.go | 7 +- felix/daemon/daemon.go | 4 +- kube-controllers/cmd/kube-controllers/main.go | 4 +- libcalico-go/lib/backend/k8s/k8s.go | 6 ++ .../upgrade/migrator/clients/v1/k8s/k8s.go | 14 +++- libcalico-go/lib/winutils/winutils.go | 80 ++++++++++++++++++- node/Dockerfile-windows | 2 +- node/pkg/cni/token_watch.go | 6 +- node/pkg/lifecycle/shutdown/shutdown.go | 4 +- node/pkg/lifecycle/startup/startup.go | 6 +- node/pkg/winupgrade/should_upgrade.go | 4 +- node/pkg/winupgrade/upgrade.go | 3 +- typha/pkg/discovery/discovery.go | 4 +- typha/pkg/k8s/lookup.go | 4 +- 20 files changed, 143 insertions(+), 83 deletions(-) diff --git a/apiserver/cmd/apiserver/server/watch.go b/apiserver/cmd/apiserver/server/watch.go index 1a13fdb9eea..2c85a7ebf5f 100644 --- a/apiserver/cmd/apiserver/server/watch.go +++ b/apiserver/cmd/apiserver/server/watch.go @@ -7,12 +7,11 @@ import ( "os" "time" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" - "k8s.io/client-go/tools/clientcmd" ) const ( @@ -30,10 +29,10 @@ func WatchExtensionAuth(stopChan chan struct{}) (bool, error) { // set up k8s client // attempt 1: KUBECONFIG env var cfgFile := os.Getenv("KUBECONFIG") - cfg, err := clientcmd.BuildConfigFromFlags("", cfgFile) + cfg, err := winutils.BuildConfigFromFlags("", cfgFile) if err != nil { // attempt 2: in cluster config - if cfg, err = rest.InClusterConfig(); err != nil { + if cfg, err = winutils.GetInClusterConfig(); err != nil { return false, err } } diff --git a/cni-plugin/Dockerfile-windows b/cni-plugin/Dockerfile-windows index cd40b2e9641..745a77b1ad2 100644 --- a/cni-plugin/Dockerfile-windows +++ b/cni-plugin/Dockerfile-windows @@ -34,6 +34,6 @@ ADD LICENSE /licenses/ ADD bin/windows/ /opt/cni/bin/ -ENV PATH=$PATH:/opt/cni/bin +ENV PATH=$PATH;/opt/cni/bin WORKDIR /opt/cni/bin CMD ["/opt/cni/bin/install.exe"] diff --git a/cni-plugin/pkg/install/install.go b/cni-plugin/pkg/install/install.go index 8cffad330d5..6542de7dd1c 100644 --- a/cni-plugin/pkg/install/install.go +++ b/cni-plugin/pkg/install/install.go @@ -20,7 +20,6 @@ import ( "encoding/json" "fmt" "io" - "net" "os" "os/exec" "runtime" @@ -34,7 +33,6 @@ import ( "go.etcd.io/etcd/client/pkg/v3/fileutil" "k8s.io/client-go/rest" - certutil "k8s.io/client-go/util/cert" "github.com/projectcalico/calico/libcalico-go/lib/logutils" "github.com/projectcalico/calico/libcalico-go/lib/names" @@ -146,7 +144,7 @@ func Install() error { if fileExists(serviceAccountTokenFile) { logrus.Info("Running as a Kubernetes pod") // FIXME: get rid of this and call rest.InClusterConfig() directly when containerd v1.6 is EOL'd - kubecfg, err = getInClusterConfig() + kubecfg, err = winutils.GetInClusterConfig() if err != nil { return err } @@ -611,40 +609,3 @@ func destinationUptoDate(src, dst string) (bool, error) { // slice of bytes. } } - -// FIXME: get rid of this and call rest.InClusterConfig() directly when containerd v1.6 is EOL'd -// getInClusterConfig returns a config object which uses the service account -// kubernetes gives to pods. It's intended for clients that expect to be -// running inside a pod running on kubernetes. It will return ErrNotInCluster -// if called from a process not running in a kubernetes environment. -// It is a copy of InClusterConfig() from k8s.io/client-go/rest but using -// winutils.GetHostPath() for the file paths, so that Windows hostprocess -// containers on containerd v1.6 can work with the in-cluster config. -func getInClusterConfig() (*rest.Config, error) { - tokenFile := winutils.GetHostPath("/var/run/secrets/kubernetes.io/serviceaccount/token") - rootCAFile := winutils.GetHostPath("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt") - host, port := os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT") - if len(host) == 0 || len(port) == 0 { - return nil, rest.ErrNotInCluster - } - - token, err := os.ReadFile(tokenFile) - if err != nil { - return nil, err - } - - tlsClientConfig := rest.TLSClientConfig{} - - if _, err := certutil.NewPool(rootCAFile); err != nil { - logrus.Errorf("Expected to load root CA config from %s, but got err: %v", rootCAFile, err) - } else { - tlsClientConfig.CAFile = rootCAFile - } - - return &rest.Config{ - Host: "https://" + net.JoinHostPort(host, port), - TLSClientConfig: tlsClientConfig, - BearerToken: string(token), - BearerTokenFile: tokenFile, - }, nil -} diff --git a/cni-plugin/pkg/k8s/k8s.go b/cni-plugin/pkg/k8s/k8s.go index 124f1950023..733267f552c 100644 --- a/cni-plugin/pkg/k8s/k8s.go +++ b/cni-plugin/pkg/k8s/k8s.go @@ -30,10 +30,12 @@ import ( "github.com/containernetworking/plugins/pkg/ipam" libipam "github.com/projectcalico/calico/libcalico-go/lib/ipam" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" libapi "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" @@ -868,9 +870,18 @@ func NewK8sClient(conf types.NetConf, logger *logrus.Entry) (*kubernetes.Clients } // Use the kubernetes client code to load the kubeconfig file and combine it with the overrides. - config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( - &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig}, - configOverrides).ClientConfig() + var config *rest.Config + var err error + if winutils.InHostProcessContainer() { + // ClientConfig() calls InClusterConfig() at some point, which doesn't work + // on Windows HPC. Use winutils.GetInClusterConfig() instead in this case. + // FIXME: this will no longer be needed when containerd v1.6 is EOL'd + config, err = winutils.GetInClusterConfig() + } else { + config, err = clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig}, + configOverrides).ClientConfig() + } if err != nil { return nil, err } diff --git a/cni-plugin/pkg/plugin/plugin.go b/cni-plugin/pkg/plugin/plugin.go index 222d09c24b8..f9f22125839 100644 --- a/cni-plugin/pkg/plugin/plugin.go +++ b/cni-plugin/pkg/plugin/plugin.go @@ -37,9 +37,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" "github.com/projectcalico/calico/libcalico-go/lib/seedrng" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" @@ -92,7 +92,7 @@ func testConnection() error { // If we have a kubeconfig, test connection to the APIServer if conf.Kubernetes.Kubeconfig != "" { - k8sconfig, err := clientcmd.BuildConfigFromFlags("", conf.Kubernetes.Kubeconfig) + k8sconfig, err := winutils.BuildConfigFromFlags("", conf.Kubernetes.Kubeconfig) if err != nil { return fmt.Errorf("error building K8s client config: %s", err) } diff --git a/confd/pkg/backends/calico/routes.go b/confd/pkg/backends/calico/routes.go index 2ad321b7b99..551f3f991bf 100644 --- a/confd/pkg/backends/calico/routes.go +++ b/confd/pkg/backends/calico/routes.go @@ -25,11 +25,10 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" - "k8s.io/client-go/tools/clientcmd" "github.com/projectcalico/calico/confd/pkg/resource/template" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" ) const ( @@ -73,11 +72,11 @@ func NewRouteGenerator(c *client) (rg *routeGenerator, err error) { // set up k8s client // attempt 1: KUBECONFIG env var cfgFile := os.Getenv("KUBECONFIG") - cfg, err := clientcmd.BuildConfigFromFlags("", cfgFile) + cfg, err := winutils.BuildConfigFromFlags("", cfgFile) if err != nil { log.WithError(err).Info("KUBECONFIG environment variable not found, attempting in-cluster") // attempt 2: in cluster config - if cfg, err = rest.InClusterConfig(); err != nil { + if cfg, err = winutils.GetInClusterConfig(); err != nil { return } } diff --git a/confd/pkg/backends/calico/secret_watcher.go b/confd/pkg/backends/calico/secret_watcher.go index 73b347eecf2..9267f6502e7 100644 --- a/confd/pkg/backends/calico/secret_watcher.go +++ b/confd/pkg/backends/calico/secret_watcher.go @@ -19,13 +19,12 @@ import ( "sync" "time" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" - "k8s.io/client-go/tools/clientcmd" ) type secretWatchData struct { @@ -63,11 +62,11 @@ func NewSecretWatcher(c *client) (*secretWatcher, error) { // set up k8s client // attempt 1: KUBECONFIG env var cfgFile := os.Getenv("KUBECONFIG") - cfg, err := clientcmd.BuildConfigFromFlags("", cfgFile) + cfg, err := winutils.BuildConfigFromFlags("", cfgFile) if err != nil { log.WithError(err).Info("KUBECONFIG environment variable not found, attempting in-cluster") // attempt 2: in cluster config - if cfg, err = rest.InClusterConfig(); err != nil { + if cfg, err = winutils.GetInClusterConfig(); err != nil { return nil, err } } diff --git a/felix/daemon/daemon.go b/felix/daemon/daemon.go index 56a1e189fe8..83ec939340c 100644 --- a/felix/daemon/daemon.go +++ b/felix/daemon/daemon.go @@ -31,9 +31,9 @@ import ( "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" "github.com/projectcalico/calico/libcalico-go/lib/backend/syncersv1/dedupebuffer" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" "github.com/projectcalico/calico/libcalico-go/lib/seedrng" @@ -309,7 +309,7 @@ configRetry: } else { // Not using KDD, fall back on trying to get a Kubernetes client from the environment. log.Info("Not using Kubernetes datastore driver, trying to get a Kubernetes client...") - k8sconf, err := rest.InClusterConfig() + k8sconf, err := winutils.GetInClusterConfig() if err != nil { log.WithError(err).Info("Kubernetes in-cluster config not available. " + "Assuming we're not in a Kubernetes deployment.") diff --git a/kube-controllers/cmd/kube-controllers/main.go b/kube-controllers/cmd/kube-controllers/main.go index 8d3e30d72cb..b225ab1c027 100644 --- a/kube-controllers/cmd/kube-controllers/main.go +++ b/kube-controllers/cmd/kube-controllers/main.go @@ -28,6 +28,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/projectcalico/calico/libcalico-go/lib/seedrng" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" log "github.com/sirupsen/logrus" "go.etcd.io/etcd/client/pkg/v3/srv" @@ -37,7 +38,6 @@ import ( "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" - "k8s.io/client-go/tools/clientcmd" "k8s.io/klog/v2" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" @@ -359,7 +359,7 @@ func getClients(kubeconfig string) (*kubernetes.Clientset, client.Interface, err // Now build the Kubernetes client, we support in-cluster config and kubeconfig // as means of configuring the client. - k8sconfig, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + k8sconfig, err := winutils.BuildConfigFromFlags("", kubeconfig) if err != nil { return nil, nil, fmt.Errorf("failed to build kubernetes client config: %s", err) } diff --git a/libcalico-go/lib/backend/k8s/k8s.go b/libcalico-go/lib/backend/k8s/k8s.go index 7f48ff33fb9..4712f01b5ec 100644 --- a/libcalico-go/lib/backend/k8s/k8s.go +++ b/libcalico-go/lib/backend/k8s/k8s.go @@ -36,6 +36,7 @@ import ( "github.com/projectcalico/calico/libcalico-go/lib/backend/model" cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" "github.com/projectcalico/calico/libcalico-go/lib/net" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -335,6 +336,11 @@ func CreateKubernetesClientset(ca *apiconfig.CalicoAPIConfigSpec) (*rest.Config, return nil, nil, resources.K8sErrorToCalico(err, nil) } config, err = clientConfig.ClientConfig() + } else if winutils.InHostProcessContainer() { + // ClientConfig() calls InClusterConfig() at some point, which doesn't work + // on Windows HPC. Use winutils.GetInClusterConfig() instead in this case. + // FIXME: this will no longer be needed when containerd v1.6 is EOL'd + config, err = winutils.GetInClusterConfig() } else { config, err = clientcmd.NewNonInteractiveDeferredLoadingClientConfig( &loadingRules, configOverrides).ClientConfig() diff --git a/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/k8s.go b/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/k8s.go index 394bdaed3f3..4848103d937 100644 --- a/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/k8s.go +++ b/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/k8s.go @@ -35,6 +35,7 @@ import ( "github.com/projectcalico/calico/libcalico-go/lib/errors" "github.com/projectcalico/calico/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/custom" "github.com/projectcalico/calico/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" ) type KubeClient struct { @@ -84,8 +85,17 @@ func NewKubeClient(kc *capi.KubeConfig) (*KubeClient, error) { // A kubeconfig file was provided. Use it to load a config, passing through // any overrides. - config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( - &loadingRules, configOverrides).ClientConfig() + var config *rest.Config + var err error + if winutils.InHostProcessContainer() { + // ClientConfig() calls InClusterConfig() at some point, which doesn't work + // on Windows HPC. Use winutils.GetInClusterConfig() instead in this case. + // FIXME: this will no longer be needed when containerd v1.6 is EOL'd + config, err = winutils.GetInClusterConfig() + } else { + config, err = clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + &loadingRules, configOverrides).ClientConfig() + } if err != nil { return nil, resources.K8sErrorToCalico(err, nil) } diff --git a/libcalico-go/lib/winutils/winutils.go b/libcalico-go/lib/winutils/winutils.go index bdf9546ea17..e03ffcf72bb 100644 --- a/libcalico-go/lib/winutils/winutils.go +++ b/libcalico-go/lib/winutils/winutils.go @@ -15,11 +15,19 @@ package winutils import ( "bytes" + "net" "os" "os/exec" "path/filepath" "runtime" "strings" + + "github.com/sirupsen/logrus" + + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + certutil "k8s.io/client-go/util/cert" ) func Powershell(args ...string) (string, string, error) { @@ -51,13 +59,23 @@ func Powershell(args ...string) (string, string, error) { return stdout.String(), stderr.String(), err } +// InHostProcessContainer returns true if inside a Windows HostProcess container, by +// checking if OS is Windows and the $env:CONTAINER_SANDBOX_MOUNT_POINT env variable +// is set. +func InHostProcessContainer() bool { + if runtime.GOOS == "windows" && os.Getenv("CONTAINER_SANDBOX_MOUNT_POINT") != "" { + return true + } + return false +} + // GetHostPath returns the mount paths for a container // In the case of Windows HostProcess containers this prepends the CONTAINER_SANDBOX_MOUNT_POINT env variable // for other operating systems or if the sandbox env variable is not set it returns the standard mount points // see https://kubernetes.io/docs/tasks/configure-pod-container/create-hostprocess-pod/#volume-mounts // FIXME: this will no longer be needed when containerd v1.6 is EOL'd func GetHostPath(path string) string { - if runtime.GOOS == "windows" { + if InHostProcessContainer() { sandbox := os.Getenv("CONTAINER_SANDBOX_MOUNT_POINT") // join them and return with forward slashes so it can be serialized properly in json later if required path := strings.TrimLeft(path, "c:") @@ -67,3 +85,63 @@ func GetHostPath(path string) string { } return path } + +// FIXME: get rid of this and call rest.InClusterConfig() directly when containerd v1.6 is EOL'd +// GetInClusterConfig returns a config object which uses the service account +// kubernetes gives to pods. It's intended for clients that expect to be +// running inside a pod running on kubernetes. It will return ErrNotInCluster +// if called from a process not running in a kubernetes environment. +// It is a copy of InClusterConfig() from k8s.io/client-go/rest but using +// winutils.GetHostPath() for the file paths, so that Windows hostprocess +// containers on containerd v1.6 can work with the in-cluster config. +func GetInClusterConfig() (*rest.Config, error) { + tokenFile := GetHostPath("/var/run/secrets/kubernetes.io/serviceaccount/token") + rootCAFile := GetHostPath("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt") + host, port := os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT") + if len(host) == 0 || len(port) == 0 { + return nil, rest.ErrNotInCluster + } + + token, err := os.ReadFile(tokenFile) + if err != nil { + return nil, err + } + + tlsClientConfig := rest.TLSClientConfig{} + + if _, err := certutil.NewPool(rootCAFile); err != nil { + logrus.Errorf("Expected to load root CA config from %s, but got err: %v", rootCAFile, err) + } else { + tlsClientConfig.CAFile = rootCAFile + } + + return &rest.Config{ + Host: "https://" + net.JoinHostPort(host, port), + TLSClientConfig: tlsClientConfig, + BearerToken: string(token), + BearerTokenFile: tokenFile, + }, nil +} + +// FIXME: get rid of this and call clientcmd.BuildConfigFromFlags() directly when containerd v1.6 is EOL'd +// BuildConfigFromFlags is a helper function that builds configs from a master +// url or a kubeconfig filepath. These are passed in as command line flags for cluster +// components. Warnings should reflect this usage. If neither masterUrl or kubeconfigPath +// are passed in we fallback to inClusterConfig. If inClusterConfig fails, we fallback +// to the default config. +// It is a copy of BuildConfigFromFlags() from k8s.io/client-go/tools/clientcmd but using +// GetInClusterConfig(), which uses winutils.GetHostPath() for the file paths, so that +// Windows hostprocess containers on containerd v1.6 can work with the in-cluster config. +func BuildConfigFromFlags(masterUrl, kubeconfigPath string) (*rest.Config, error) { + if kubeconfigPath == "" && masterUrl == "" { + logrus.Warning("Neither --kubeconfig nor --master was specified. Using the inClusterConfig. This might not work.") + kubeconfig, err := GetInClusterConfig() + if err == nil { + return kubeconfig, nil + } + logrus.Warning("error creating inClusterConfig, falling back to default config: ", err) + } + return clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfigPath}, + &clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: masterUrl}}).ClientConfig() +} diff --git a/node/Dockerfile-windows b/node/Dockerfile-windows index e9876ba83da..f3c154b0261 100644 --- a/node/Dockerfile-windows +++ b/node/Dockerfile-windows @@ -42,7 +42,7 @@ ADD windows-packaging/CalicoWindows/libs/hns/hns.psm1 /CalicoWindows/libs/hns/hn ADD windows-packaging/nssm.exe /nssm.exe ADD windows-packaging/CalicoWindows/uninstall-calico-hpc.ps1 /uninstall-calico.ps1 -ENV PATH=$PATH:/CalicoWindows +ENV PATH=$PATH;/CalicoWindows WORKDIR /CalicoWindows # The nanoserver image does not have powershell but this works because # this container will be running on the host. diff --git a/node/pkg/cni/token_watch.go b/node/pkg/cni/token_watch.go index 0c58732f644..de0e788ab20 100644 --- a/node/pkg/cni/token_watch.go +++ b/node/pkg/cni/token_watch.go @@ -18,7 +18,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" "github.com/projectcalico/calico/libcalico-go/lib/winutils" ) @@ -65,7 +64,7 @@ func NamespaceOfUsedServiceAccount() string { } func BuildClientSet() (*kubernetes.Clientset, error) { - cfg, err := clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")) + cfg, err := winutils.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")) if err != nil { return nil, err } @@ -225,7 +224,7 @@ func Run() { for tu := range tokenChan { logrus.Info("Update of CNI kubeconfig triggered based on elapsed time.") - cfg, err := clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")) + cfg, err := winutils.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")) if err != nil { logrus.WithError(err).Error("Error generating kube config.") continue @@ -281,6 +280,5 @@ current-context: calico-context` logrus.WithError(err).Error("Failed to write CNI plugin kubeconfig file") return } - // logrus.WithField("path", kubeconfigPath).Info("Wrote updated CNI kubeconfig file.") logrus.WithField("path", winutils.GetHostPath(kubeconfigPath)).Info("Wrote updated CNI kubeconfig file.") } diff --git a/node/pkg/lifecycle/shutdown/shutdown.go b/node/pkg/lifecycle/shutdown/shutdown.go index 21a3fc06cd7..ba4499231da 100644 --- a/node/pkg/lifecycle/shutdown/shutdown.go +++ b/node/pkg/lifecycle/shutdown/shutdown.go @@ -19,8 +19,8 @@ import ( log "github.com/sirupsen/logrus" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" "github.com/projectcalico/calico/node/pkg/lifecycle/utils" ) @@ -43,7 +43,7 @@ func Run() { var clientset *kubernetes.Clientset // If running under kubernetes with secrets to call k8s API - if config, err := clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")); err == nil { + if config, err := winutils.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")); err == nil { // default timeout is 30 seconds, which isn't appropriate for this kind of // shutdown action because network services, like kube-proxy might not be // running and we don't want to block the full 30 seconds if they are just diff --git a/node/pkg/lifecycle/startup/startup.go b/node/pkg/lifecycle/startup/startup.go index b2e9e11ed87..452ac45556c 100644 --- a/node/pkg/lifecycle/startup/startup.go +++ b/node/pkg/lifecycle/startup/startup.go @@ -31,7 +31,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" @@ -45,6 +44,7 @@ import ( "github.com/projectcalico/calico/libcalico-go/lib/selector" "github.com/projectcalico/calico/libcalico-go/lib/upgrade/migrator" "github.com/projectcalico/calico/libcalico-go/lib/upgrade/migrator/clients" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" "github.com/projectcalico/calico/node/pkg/calicoclient" "github.com/projectcalico/calico/node/pkg/lifecycle/startup/autodetection" @@ -130,7 +130,7 @@ func Run() { } // If running under kubernetes with secrets to call k8s API - if config, err := clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")); err == nil { + if config, err := winutils.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")); err == nil { // default timeout is 30 seconds, which isn't appropriate for this kind of // startup action because network services, like kube-proxy might not be // running and we don't want to block the full 30 seconds if they are just @@ -330,7 +330,7 @@ func MonitorIPAddressSubnets() { if nodeRef := os.Getenv("CALICO_K8S_NODE_REF"); nodeRef != "" { k8sNodeName = nodeRef } - if config, err = clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")); err == nil { + if config, err = winutils.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")); err == nil { // Create the k8s clientset. clientset, err = kubernetes.NewForConfig(config) if err != nil { diff --git a/node/pkg/winupgrade/should_upgrade.go b/node/pkg/winupgrade/should_upgrade.go index b2904cc7a07..a3410ae5c9c 100644 --- a/node/pkg/winupgrade/should_upgrade.go +++ b/node/pkg/winupgrade/should_upgrade.go @@ -19,9 +19,9 @@ import ( "strings" "github.com/projectcalico/calico/libcalico-go/lib/names" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" "github.com/projectcalico/calico/node/pkg/lifecycle/utils" @@ -37,7 +37,7 @@ func ShouldInstallUpgradeService() { nodeName := determineNodeName() log.Debugf("Check if Calico upgrade service should be installed on node: %s. Version: %s, Variant: %s, baseDir: %s", nodeName, version, variant, baseDir()) - config, err := clientcmd.BuildConfigFromFlags("", kubeConfigFile()) + config, err := winutils.BuildConfigFromFlags("", kubeConfigFile()) if err != nil { log.WithError(err).Fatal("Failed to build Kubernetes client config") os.Exit(2) diff --git a/node/pkg/winupgrade/upgrade.go b/node/pkg/winupgrade/upgrade.go index 34fa435c726..e89af7bd2c9 100644 --- a/node/pkg/winupgrade/upgrade.go +++ b/node/pkg/winupgrade/upgrade.go @@ -30,7 +30,6 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" "github.com/projectcalico/calico/libcalico-go/lib/winutils" "github.com/projectcalico/calico/node/pkg/lifecycle/utils" @@ -73,7 +72,7 @@ func Run() { nodeName := utils.DetermineNodeName() log.Infof("Starting Calico upgrade service on node: %s. Version: %s, Variant: %s, baseDir: %s", nodeName, version, variant, baseDir()) - config, err := clientcmd.BuildConfigFromFlags("", kubeConfigFile()) + config, err := winutils.BuildConfigFromFlags("", kubeConfigFile()) if err != nil { log.WithError(err).Fatal("Failed to build Kubernetes client config") } diff --git a/typha/pkg/discovery/discovery.go b/typha/pkg/discovery/discovery.go index 654b6a04ca1..4a503a78cfb 100644 --- a/typha/pkg/discovery/discovery.go +++ b/typha/pkg/discovery/discovery.go @@ -24,9 +24,9 @@ import ( "github.com/sirupsen/logrus" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" "github.com/projectcalico/calico/libcalico-go/lib/set" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" "github.com/projectcalico/calico/libcalico-go/lib/seedrng" ) @@ -177,7 +177,7 @@ func (d *Discoverer) discoverTyphaAddrs() ([]Typha, error) { if d.k8sClient == nil && d.inCluster { // Client didn't provide a kube client but we're allowed to create one. logrus.Info("Creating Kubernetes client for Typha discovery...") - k8sConf, err := rest.InClusterConfig() + k8sConf, err := winutils.GetInClusterConfig() if err != nil { logrus.WithError(err).Error("Unable to create in-cluster Kubernetes config.") return nil, err diff --git a/typha/pkg/k8s/lookup.go b/typha/pkg/k8s/lookup.go index 340c86d23ab..c6242e2fb3a 100644 --- a/typha/pkg/k8s/lookup.go +++ b/typha/pkg/k8s/lookup.go @@ -20,9 +20,9 @@ import ( log "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" "github.com/projectcalico/calico/libcalico-go/lib/set" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" "github.com/projectcalico/calico/typha/pkg/calc" ) @@ -38,7 +38,7 @@ type RealK8sAPI struct { func (r *RealK8sAPI) clientSet() (*kubernetes.Clientset, error) { if r.cachedClientSet == nil { // TODO Typha: support Typha lookup without using rest.InClusterConfig(). - k8sconf, err := rest.InClusterConfig() + k8sconf, err := winutils.GetInClusterConfig() if err != nil { log.WithError(err).Error("Unable to create Kubernetes config.") return nil, err From a7021e6f53e9ca87063a7d27257552461b5b4d39 Mon Sep 17 00:00:00 2001 From: Pedro Coutinho Date: Wed, 30 Aug 2023 12:58:59 -0700 Subject: [PATCH 12/16] Remove winupgrade code --- node/cmd/calico-node/main.go | 9 - node/pkg/winupgrade/k8s_resources.go | 73 ---- node/pkg/winupgrade/should_upgrade.go | 98 ------ node/pkg/winupgrade/upgrade.go | 387 ---------------------- node/pkg/winupgrade/upgrade_suite_test.go | 36 -- node/pkg/winupgrade/upgrade_test.go | 53 --- 6 files changed, 656 deletions(-) delete mode 100644 node/pkg/winupgrade/k8s_resources.go delete mode 100644 node/pkg/winupgrade/should_upgrade.go delete mode 100644 node/pkg/winupgrade/upgrade.go delete mode 100644 node/pkg/winupgrade/upgrade_suite_test.go delete mode 100644 node/pkg/winupgrade/upgrade_test.go diff --git a/node/cmd/calico-node/main.go b/node/cmd/calico-node/main.go index 7138b21e2c5..7f644a1b1ad 100644 --- a/node/cmd/calico-node/main.go +++ b/node/cmd/calico-node/main.go @@ -38,7 +38,6 @@ import ( "github.com/projectcalico/calico/node/pkg/lifecycle/shutdown" "github.com/projectcalico/calico/node/pkg/lifecycle/startup" "github.com/projectcalico/calico/node/pkg/status" - "github.com/projectcalico/calico/node/pkg/winupgrade" ) // Create a new flag set. @@ -51,8 +50,6 @@ var runBPF = flagSet.Bool("bpf", false, "Run BPF debug tool") var runInit = flagSet.Bool("init", false, "Do privileged initialisation of a new node (mount file systems etc).") var bestEffort = flagSet.Bool("best-effort", false, "Used in combination with the init flag. Report errors but do not fail if an error occures during initialisation.") var runStartup = flagSet.Bool("startup", false, "Do non-privileged start-up routine.") -var runWinUpgrade = flagSet.Bool("upgrade-windows", false, "Run Windows upgrade service.") -var runShouldInstallWindowsUpgrade = flagSet.Bool("should-install-windows-upgrade", false, "Check if Windows upgrade service should be installed.") var runShutdown = flagSet.Bool("shutdown", false, "Do shutdown routine.") var monitorAddrs = flagSet.Bool("monitor-addresses", false, "Monitor change in node IP addresses") var runAllocateTunnelAddrs = flagSet.Bool("allocate-tunnel-addrs", false, "Configure tunnel addresses for this node") @@ -147,12 +144,6 @@ func main() { } else if *runShutdown { logrus.SetFormatter(&logutils.Formatter{Component: "shutdown"}) shutdown.Run() - } else if *runWinUpgrade { - logrus.SetFormatter(&logutils.Formatter{Component: "windows-upgrade"}) - winupgrade.Run() - } else if *runShouldInstallWindowsUpgrade { - logrus.SetFormatter(&logutils.Formatter{Component: "should-install-windows-upgrade"}) - winupgrade.ShouldInstallUpgradeService() } else if *monitorAddrs { logrus.SetFormatter(&logutils.Formatter{Component: "monitor-addresses"}) startup.ConfigureLogging() diff --git a/node/pkg/winupgrade/k8s_resources.go b/node/pkg/winupgrade/k8s_resources.go deleted file mode 100644 index f7da6050c83..00000000000 --- a/node/pkg/winupgrade/k8s_resources.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright (c) 2021 Tigera, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package winupgrade - -import ( - "context" - "time" - - apierrs "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" -) - -// k8snode holds a collection of helper functions for Kubernetes node. -type k8snode string - -// Add / remove node annotations to node. Perform Get/Check/Update so that it always working on latest version. -// If node labels has been set already, do nothing. -func (n k8snode) addRemoveNodeAnnotations(k8sClientset *kubernetes.Clientset, - toAdd map[string]string, - toRemove []string) error { - nodeName := string(n) - return wait.PollImmediate(3*time.Second, 1*time.Minute, func() (bool, error) { - node, err := k8sClientset.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{}) - if err != nil { - return false, err - } - - needUpdate := false - for k, v := range toAdd { - if currentVal, ok := node.Annotations[k]; ok && currentVal == v { - continue - } - node.Annotations[k] = v - needUpdate = true - } - - for _, k := range toRemove { - if _, ok := node.Annotations[k]; ok { - delete(node.Annotations, k) - needUpdate = true - } - } - - if needUpdate { - _, err := k8sClientset.CoreV1().Nodes().UpdateStatus(context.Background(), node, metav1.UpdateOptions{}) - if err == nil { - return true, nil - } - if !apierrs.IsConflict(err) { - return false, err - } - - // Retry on update conflicts. - return false, nil - } - - // no update needed - return true, nil - }) -} diff --git a/node/pkg/winupgrade/should_upgrade.go b/node/pkg/winupgrade/should_upgrade.go deleted file mode 100644 index a3410ae5c9c..00000000000 --- a/node/pkg/winupgrade/should_upgrade.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (c) 2021 Tigera, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package winupgrade - -import ( - "context" - "os" - "strings" - - "github.com/projectcalico/calico/libcalico-go/lib/names" - "github.com/projectcalico/calico/libcalico-go/lib/winutils" - - "k8s.io/client-go/kubernetes" - - "github.com/projectcalico/calico/node/pkg/lifecycle/utils" - - log "github.com/sirupsen/logrus" -) - -// Exit with code zero if Windows upgrade service should be installed. -func ShouldInstallUpgradeService() { - version := getVersion() - variant := getVariant() - - // Determine the name for this node. - nodeName := determineNodeName() - log.Debugf("Check if Calico upgrade service should be installed on node: %s. Version: %s, Variant: %s, baseDir: %s", nodeName, version, variant, baseDir()) - - config, err := winutils.BuildConfigFromFlags("", kubeConfigFile()) - if err != nil { - log.WithError(err).Fatal("Failed to build Kubernetes client config") - os.Exit(2) - } - clientSet, err := kubernetes.NewForConfig(config) - if err != nil { - log.WithError(err).Fatal("Failed to create Kubernetes client") - os.Exit(2) - } - - // Update annotations for running variant and version. - node := k8snode(nodeName) - err = node.addRemoveNodeAnnotations(clientSet, - map[string]string{ - CalicoVersionAnnotation: version, - CalicoVariantAnnotation: variant, - }, - []string{}) - if err != nil { - log.WithError(err).Fatal("Failed to set version/variant annotations") - os.Exit(2) - } - - upgrade, _ := upgradeTriggered(context.Background(), clientSet, nodeName) - if !upgrade { - os.Exit(1) - } - os.Exit(0) -} - -// DetermineNodeName is copied from utils package but with logging in DEBUG level. -func determineNodeName() string { - var nodeName string - var err error - - // Determine the name of this node. Precedence is: - // - NODENAME - // - Value stored in our nodename file. - // - HOSTNAME (lowercase) - // - os.Hostname (lowercase). - // We use the names.Hostname which lowercases and trims the name. - if nodeName = strings.TrimSpace(os.Getenv("NODENAME")); nodeName != "" { - log.Debugf("Using NODENAME environment for node name %s", nodeName) - } else if nodeName = utils.NodenameFromFile(); nodeName != "" { - log.Debugf("Using stored node name %s", nodeName) - } else if nodeName = strings.ToLower(strings.TrimSpace(os.Getenv("HOSTNAME"))); nodeName != "" { - log.Debugf("Using HOSTNAME environment (lowercase) for node name %s", nodeName) - } else if nodeName, err = names.Hostname(); err != nil { - log.WithError(err).Error("Unable to determine hostname") - utils.Terminate() - } else { - log.Warn("Using auto-detected node name. It is recommended that an explicit value is supplied using " + - "the NODENAME environment variable.") - } - log.Debugf("Determined node name: %s", nodeName) - - return nodeName -} diff --git a/node/pkg/winupgrade/upgrade.go b/node/pkg/winupgrade/upgrade.go deleted file mode 100644 index e89af7bd2c9..00000000000 --- a/node/pkg/winupgrade/upgrade.go +++ /dev/null @@ -1,387 +0,0 @@ -// Copyright (c) 2021 Tigera, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package winupgrade - -import ( - "context" - "fmt" - "os" - "os/signal" - "path/filepath" - "strings" - "syscall" - "time" - - image "github.com/distribution/distribution/reference" - - "github.com/projectcalico/calico/node/pkg/lifecycle/startup" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - - "github.com/projectcalico/calico/libcalico-go/lib/winutils" - "github.com/projectcalico/calico/node/pkg/lifecycle/utils" - - log "github.com/sirupsen/logrus" -) - -const ( - CalicoKubeConfigFile = "calico-kube-config" - CalicoUpgradeDir = "c:\\CalicoUpgrade" - EnterpriseDir = "TigeraCalico" - CalicoUpgradeLabel = "projectcalico.org/windows-upgrade" - CalicoUpgradeInProgress = "in-progress" - CalicoVersionAnnotation = "projectcalico.org/version" - CalicoVariantAnnotation = "projectcalico.org/variant" - CalicoUpgradeScript = "calico-upgrade.ps1" -) - -func getVariant() string { - if runningEnterprise() { - return "TigeraSecureEnterprise" - } - return "Calico" -} - -func getVersion() string { - return startup.VERSION -} - -// This file contains the upgrade processing for the calico/node. This -// includes: -// - Monitoring node labels and getting the Calico Windows upgrade script file from the label. -// - Uninstalling current Calico Windows (OSS or Enterprise) running on the node. -// - Install new version of Calico Windows. -func Run() { - version := getVersion() - variant := getVariant() - - // Determine the name for this node. - nodeName := utils.DetermineNodeName() - log.Infof("Starting Calico upgrade service on node: %s. Version: %s, Variant: %s, baseDir: %s", nodeName, version, variant, baseDir()) - - config, err := winutils.BuildConfigFromFlags("", kubeConfigFile()) - if err != nil { - log.WithError(err).Fatal("Failed to build Kubernetes client config") - } - clientSet, err := kubernetes.NewForConfig(config) - if err != nil { - log.WithError(err).Fatal("Failed to create Kubernetes client") - } - - stdout, stderr, err := winutils.Powershell("Get-ComputerInfo | select WindowsVersion, OsBuildNumber, OsHardwareAbstractionLayer") - fmt.Println(stdout, stderr) - if err != nil { - log.WithError(err).Fatal("Failed to interact with powershell") - } - - ctx, cancel := context.WithCancel(context.Background()) - go loop(ctx, clientSet, nodeName) - - // Trap cancellation on Windows. https://golang.org/pkg/os/signal/ - sigCh := make(chan os.Signal, 1) - signal.Notify( - sigCh, - syscall.SIGTERM, - syscall.SIGINT, - syscall.SIGQUIT, - ) - - <-sigCh - cancel() - log.Info("Received system signal to exit") -} - -func loop(ctx context.Context, cs kubernetes.Interface, nodeName string) { - ticker := time.NewTicker(10 * time.Second) - upgradeScript := filepath.Join(CalicoUpgradeDir, CalicoUpgradeScript) - - for { - select { - case <-ctx.Done(): - ticker.Stop() - return - case <-ticker.C: - upgrade, err := upgradeTriggered(ctx, cs, nodeName) - if err != nil { - log.WithError(err).Error("Failed to check node upgrade status, will retry") - break - } - // If upgrade not triggered yet just silently continue. - if !upgrade { - break - } - - if !pathExists(upgradeScript) { - log.Info("Upgrade triggered but upgrade artifacts not ready yet, will retry") - break - } - - log.Info("Calico upgrade process is starting") - - // Before executing the script, verify host path volume mount. - err = verifyPodImageWithHostPathVolume(cs, nodeName, CalicoUpgradeDir) - if err != nil { - log.WithError(err).Fatal("Failed to verify windows-upgrade pod image") - } - - err = uninstall() - if err != nil { - log.WithError(err).Error("Uninstall failed, will retry") - break - } - - time.Sleep(3 * time.Second) - err = execScript(upgradeScript) - if err != nil { - log.WithError(err).Fatal("Failed to upgrade to new version") - } - - // Upgrade will run in another process. The running - // calico-upgrade service is done. The new calico-upgrade - // service will clean the old service up. - date := time.Now().Format("2006-01-02") - log.Info(fmt.Sprintf("Upgrade is in progress. Upgrade log is in c:\\calico-upgrade.%v.log", date)) - time.Sleep(3 * time.Second) - return - } - } -} - -func upgradeTriggered(ctx context.Context, cs kubernetes.Interface, nodeName string) (bool, error) { - node, err := cs.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) - // Return error getting node info. - if err != nil { - return false, fmt.Errorf("Could not get node resource: %w", err) - } - - upgradeStatus, ok := node.Labels[CalicoUpgradeLabel] - // Upgrade label doesn't exist yet. - if !ok { - return false, nil - } - if upgradeStatus != CalicoUpgradeInProgress { - return false, fmt.Errorf("Unexpected upgrade status label value: %v", upgradeStatus) - } - - return true, nil -} - -func pathExists(path string) bool { - if _, err := os.Stat(path); err != nil { - return false - } - - return true -} - -// Return the base directory for Calico upgrade service. -func baseDir() string { - dir := filepath.Dir(os.Args[0]) - return "c:\\" + filepath.Base(dir) -} - -// Return if the monitor service is running as part of Enterprise installation. -func runningEnterprise() bool { - return strings.Contains(baseDir(), EnterpriseDir) -} - -// Return kubeconfig file path for Calico -func kubeConfigFile() string { - return baseDir() + "\\" + CalicoKubeConfigFile -} - -func uninstall() error { - path := filepath.Join(baseDir(), "uninstall-calico.ps1") - log.Infof("Start uninstall script %s\n", path) - stdout, stderr, err := winutils.Powershell(path + " -ExceptUpgradeService $true") - fmt.Println(stdout, stderr) - if err != nil { - return err - } - // After the uninstall completes, move the existing calico-node.exe to - // a temporary file. The calico-upgrade service is still running so not - // doing this means we cannot replace calico-node.exe with the upgrade. - stdout, stderr, err = winutils.Powershell(fmt.Sprintf(`mv %v\calico-node.exe %v\calico-node.exe.to-be-replaced`, baseDir(), baseDir())) - fmt.Println(stdout, stderr) - if err != nil { - return err - } - - return nil -} - -func execScript(script string) error { - log.Infof("Start script %s\n", script) - - // This has to be done in a separate process because when the new calico services are started, the existing - // calico-upgrade service is removed so the new calico-upgrade service can be started. - // However, removing the existing calico-upgrade service means the powershell - // process running the upgrade script is killed and the installation is left - // incomplete. - cmd := fmt.Sprintf(`Start-Process powershell -argument %q -WindowStyle hidden`, script) - stdout, stderr, err := winutils.Powershell(cmd) - - if err != nil { - return err - } - fmt.Println(stdout, stderr) - return nil -} - -func verifyImagesSharePathPrefix(first, second string) error { - n1, err := image.ParseNamed(first) - if err != nil { - return err - } - n2, err := image.ParseNamed(second) - if err != nil { - return err - } - // Compare the domain parts (e.g. docker.io) of the image references. - // Domain is always present, ParseNamed fails without a domain. - if image.Domain(n1) != image.Domain(n2) { - return fmt.Errorf("images %q and %q do not share the same domain", first, second) - } - - // Split the image path. E.g. if the image is - // "docker.io/calico/node:v3.21.0" then the path is "calico/node". - n1PathParts := strings.Split(image.Path(n1), "/") - n2PathParts := strings.Split(image.Path(n2), "/") - - // Special case: if the image references are both short image references - // like my-registry.org/node:v3.21.0 then the path will just be the image - // component itself. In this case, we can only compare the domain. - if len(n1PathParts) == 1 && len(n2PathParts) == 1 { - return nil - } - - // Validation should catch this but just in case. - if len(n1PathParts) == 0 || len(n2PathParts) == 0 { - return fmt.Errorf("images %q and/or %q do not contain a valid path", first, second) - } - - // If image paths do not have equal # of segments then they don't match. - if len(n1PathParts) != len(n2PathParts) { - return fmt.Errorf("images %q and %q do not share the same path prefix", first, second) - } - - // Remove the last segment of the image path since it will contain the - // component name. - n1PathPrefix := n1PathParts[:len(n1PathParts)-1] - n2PathPrefix := n2PathParts[:len(n2PathParts)-1] - - // Compare the image path prefix (e.g. "docker.io/calico", "quay.io/calico") - // of the two images - for i := range n1PathPrefix { - if n1PathPrefix[i] != n2PathPrefix[i] { - return fmt.Errorf("images %q and %q are not from the same registry and path", first, second) - } - } - return nil -} - -func verifyPodImageWithHostPathVolume(cs kubernetes.Interface, nodeName string, hostPath string) error { - // Get pod list for calico-system pods. - list, err := cs.CoreV1().Pods("calico-system").List(context.TODO(), metav1.ListOptions{}) - if err != nil { - return err - } - - // Find the calico-node pod. - var calicoPod v1.Pod - var found bool - for _, pod := range list.Items { - if strings.HasPrefix(pod.Name, "calico-node") && pod.Spec.ServiceAccountName == "calico-node" { - calicoPod = pod - found = true - break - } - } - - if !found { - return fmt.Errorf("could not find a calico-node pod") - } - - // Ensure the pod is owned by the calico-node daemonset. - var ownedByCalicoDs bool - for _, ownerRef := range calicoPod.ObjectMeta.OwnerReferences { - if ownerRef.Kind == "DaemonSet" && ownerRef.Name == "calico-node" && *ownerRef.Controller { - ownedByCalicoDs = true - } - } - if !ownedByCalicoDs { - return fmt.Errorf("calico-node pod not owned by calico-node daemonset") - } - - // Get the calico node nodeImage - var nodeImage string - for _, c := range calicoPod.Spec.Containers { - if c.Name == "calico-node" { - nodeImage = c.Image - break - } - } - if nodeImage == "" { - return fmt.Errorf("calico-node container image not found") - } - - log.Infof("Found node container image is %v", nodeImage) - - hasHostPathVolume := func(pod v1.Pod, hostPath string) bool { - for _, v := range pod.Spec.Volumes { - path := v.HostPath - if path == nil { - continue - } - if path.Path == hostPath { - return true - } - } - return false - } - - var podWithHostPath v1.Pod - count := 0 - for _, pod := range list.Items { - // Only look at pods on this node. - if pod.Spec.NodeName != nodeName { - continue - } - if hasHostPathVolume(pod, hostPath) { - podWithHostPath = pod - count++ - } - } - - if count == 0 { - return fmt.Errorf("Failed to find pod with expected host path") - } - - if count > 1 { - return fmt.Errorf("More than one pod has expected host path") - } - - if len(podWithHostPath.Spec.Containers) != 1 { - return fmt.Errorf("Pod with hostpath volume has more than one container") - } - upgradeImage := podWithHostPath.Spec.Containers[0].Image - log.Infof("Found upgrade image: %v", upgradeImage) - err = verifyImagesSharePathPrefix(nodeImage, upgradeImage) - if err != nil { - return err - } - return nil -} diff --git a/node/pkg/winupgrade/upgrade_suite_test.go b/node/pkg/winupgrade/upgrade_suite_test.go deleted file mode 100644 index 6ba2379a458..00000000000 --- a/node/pkg/winupgrade/upgrade_suite_test.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (c) 2021 Tigera, Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package winupgrade_test - -import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - "testing" - - "github.com/onsi/ginkgo/reporters" - - "github.com/projectcalico/calico/libcalico-go/lib/testutils" -) - -func init() { - testutils.HookLogrusForGinkgo() -} - -func TestCommands(t *testing.T) { - RegisterFailHandler(Fail) - junitReporter := reporters.NewJUnitReporter("../../report/upgrade_suite.xml") - RunSpecsWithDefaultAndCustomReporters(t, "Upgrade Suite", []Reporter{junitReporter}) -} diff --git a/node/pkg/winupgrade/upgrade_test.go b/node/pkg/winupgrade/upgrade_test.go deleted file mode 100644 index d9a59837237..00000000000 --- a/node/pkg/winupgrade/upgrade_test.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (c) 2021-2022 Tigera, Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package winupgrade - -import ( - . "github.com/onsi/ginkgo/extensions/table" - . "github.com/onsi/gomega" -) - -const ( - nodeShortImageTag = "example.com/node:v3.21.0" - nodeDockerImageTag = "docker.io/calico/node:v3.21.0" - nodeDockerImageDigest = "docker.io/calico/node@sha256:1a54e9ad69451473fde398ac63a5f5696712cf38ed00f0deadc4189927b93176" - nodeQuayImageDigest = "quay.io/calico/node@sha256:bf87045cbb6c3f9ca39b9724350f728e8dab780b3e6185d413c7f232fb0452bf" - - windowsShortImageTag = "example.com/calico-windows-upgrade:v3.21.0" - windowsShortImageTag2 = "my-registry.org/calico-windows-upgrade:v3.21.0" - windowsDockerImageTag = "docker.io/calico/windows-upgrade:v3.21.0" - windowsDockerImageDigest = "docker.io/calico/windows-upgrade@sha256:1aa17a74e3f084e94b0d1f93bdd745c8c88cbb292907ac4fa94d6f206a5e49db" - windowsQuayImageTag = "quay.io/calico/windows-upgrade:v3.21.0" - - nodeLongImageTag = "example.com/tigera/calico/testing/node:v3.21.0" - windowsLongImageTag = "example.com/tigera/calico/windows-upgrade:v3.21.0" -) - -var _ = DescribeTable("verifyImagesShareRegistryPath", - func(upgradeImage string, nodeImage string, noError bool) { - err := verifyImagesSharePathPrefix(upgradeImage, nodeImage) - Expect(err == nil).To(Equal(noError)) - }, - Entry("same prefix, tag", windowsDockerImageTag, nodeDockerImageTag, true), - Entry("same prefix, digest1", windowsDockerImageTag, nodeDockerImageDigest, true), - Entry("same prefix, digest2", windowsDockerImageDigest, nodeDockerImageDigest, true), - Entry("same prefix, short tags", nodeShortImageTag, windowsShortImageTag, true), - - Entry("diff prefix, tag", windowsQuayImageTag, nodeDockerImageTag, false), - Entry("diff prefix, digest1", windowsDockerImageTag, nodeQuayImageDigest, false), - Entry("diff prefix, digest2", nodeDockerImageDigest, nodeQuayImageDigest, false), - Entry("diff prefix, short tags", nodeShortImageTag, windowsShortImageTag2, false), - Entry("diff prefix, diff lengths", nodeLongImageTag, windowsLongImageTag, false), -) From f28243adccef34b9baa523e8f48d09181f93839e Mon Sep 17 00:00:00 2001 From: Pedro Coutinho Date: Wed, 30 Aug 2023 14:25:07 -0700 Subject: [PATCH 13/16] 'make fix' on confd --- confd/pkg/backends/calico/secret_watcher.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/confd/pkg/backends/calico/secret_watcher.go b/confd/pkg/backends/calico/secret_watcher.go index 9267f6502e7..550ee17c70c 100644 --- a/confd/pkg/backends/calico/secret_watcher.go +++ b/confd/pkg/backends/calico/secret_watcher.go @@ -19,12 +19,13 @@ import ( "sync" "time" - "github.com/projectcalico/calico/libcalico-go/lib/winutils" log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" + + "github.com/projectcalico/calico/libcalico-go/lib/winutils" ) type secretWatchData struct { From 1401e4a0d8d0012d809d53c4f0f66de8b8152529 Mon Sep 17 00:00:00 2001 From: Pedro Coutinho Date: Tue, 12 Sep 2023 15:56:17 -0700 Subject: [PATCH 14/16] Fix KUBECONFIG env var bleeding from host to containers on windows. Use winutils.GetHostPath() on file paths in felix param_types.go. Add resolving of $env:CONTAINER_SANDBOX_MOUNT_POINT to winutils.GetHostPath(). --- apiserver/cmd/apiserver/server/watch.go | 6 ++++++ confd/pkg/backends/calico/routes.go | 6 ++++++ confd/pkg/backends/calico/secret_watcher.go | 6 ++++++ felix/config/param_types.go | 6 ++++++ libcalico-go/lib/winutils/winutils.go | 7 +++++-- node/pkg/cni/token_watch.go | 19 +++++++++++++++++-- node/pkg/lifecycle/shutdown/shutdown.go | 9 ++++++++- node/pkg/lifecycle/startup/startup.go | 18 ++++++++++++++++-- .../CalicoWindows/uninstall-calico-hpc.ps1 | 4 ++++ 9 files changed, 74 insertions(+), 7 deletions(-) diff --git a/apiserver/cmd/apiserver/server/watch.go b/apiserver/cmd/apiserver/server/watch.go index 2c85a7ebf5f..8825e4b25bf 100644 --- a/apiserver/cmd/apiserver/server/watch.go +++ b/apiserver/cmd/apiserver/server/watch.go @@ -29,6 +29,12 @@ func WatchExtensionAuth(stopChan chan struct{}) (bool, error) { // set up k8s client // attempt 1: KUBECONFIG env var cfgFile := os.Getenv("KUBECONFIG") + // Host env vars bleed through to the container on Windows HPC, so if cannot + // be trusted in this case + // FIXME: this will no longer be needed when containerd v1.6 is EOL'd + if winutils.InHostProcessContainer() { + cfgFile = "" + } cfg, err := winutils.BuildConfigFromFlags("", cfgFile) if err != nil { // attempt 2: in cluster config diff --git a/confd/pkg/backends/calico/routes.go b/confd/pkg/backends/calico/routes.go index 551f3f991bf..e77583c5e6f 100644 --- a/confd/pkg/backends/calico/routes.go +++ b/confd/pkg/backends/calico/routes.go @@ -72,6 +72,12 @@ func NewRouteGenerator(c *client) (rg *routeGenerator, err error) { // set up k8s client // attempt 1: KUBECONFIG env var cfgFile := os.Getenv("KUBECONFIG") + // Host env vars bleed through to the container on Windows HPC, so if cannot + // be trusted in this case + // FIXME: this will no longer be needed when containerd v1.6 is EOL'd + if winutils.InHostProcessContainer() { + cfgFile = "" + } cfg, err := winutils.BuildConfigFromFlags("", cfgFile) if err != nil { log.WithError(err).Info("KUBECONFIG environment variable not found, attempting in-cluster") diff --git a/confd/pkg/backends/calico/secret_watcher.go b/confd/pkg/backends/calico/secret_watcher.go index 550ee17c70c..9aeccdd0196 100644 --- a/confd/pkg/backends/calico/secret_watcher.go +++ b/confd/pkg/backends/calico/secret_watcher.go @@ -63,6 +63,12 @@ func NewSecretWatcher(c *client) (*secretWatcher, error) { // set up k8s client // attempt 1: KUBECONFIG env var cfgFile := os.Getenv("KUBECONFIG") + // Host env vars bleed through to the container on Windows HPC, so if cannot + // be trusted in this case + // FIXME: this will no longer be needed when containerd v1.6 is EOL'd + if winutils.InHostProcessContainer() { + cfgFile = "" + } cfg, err := winutils.BuildConfigFromFlags("", cfgFile) if err != nil { log.WithError(err).Info("KUBECONFIG environment variable not found, attempting in-cluster") diff --git a/felix/config/param_types.go b/felix/config/param_types.go index 11a936fd8de..a0cdfb1f2b8 100644 --- a/felix/config/param_types.go +++ b/felix/config/param_types.go @@ -39,6 +39,7 @@ import ( "github.com/projectcalico/calico/felix/idalloc" "github.com/projectcalico/calico/felix/stringutils" cnet "github.com/projectcalico/calico/libcalico-go/lib/net" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" ) const ( @@ -265,6 +266,11 @@ type FileParam struct { } func (p *FileParam) Parse(raw string) (interface{}, error) { + // Use GetHostPath to use/resolve the CONTAINER_SANDBOX_MOUNT_POINT env var + // if running on Windows HPC. + // FIXME: this will no longer be needed when containerd v1.6 is EOL'd + raw = winutils.GetHostPath(raw) + if p.Executable { // Special case: for executable files, we search our directory // and the system path. diff --git a/libcalico-go/lib/winutils/winutils.go b/libcalico-go/lib/winutils/winutils.go index e03ffcf72bb..d9c428253b1 100644 --- a/libcalico-go/lib/winutils/winutils.go +++ b/libcalico-go/lib/winutils/winutils.go @@ -77,9 +77,12 @@ func InHostProcessContainer() bool { func GetHostPath(path string) string { if InHostProcessContainer() { sandbox := os.Getenv("CONTAINER_SANDBOX_MOUNT_POINT") + // Remove drive letter prefixs as the CONTAINER_SANDBOX_MOUNT_POINT env var will contain it + path := strings.TrimPrefix(path, "c:") + path = strings.TrimPrefix(path, "C:") + // Remove literal unresolved CONTAINER_SANDBOX_MOUNT_POINT env var + path = strings.TrimPrefix(path, "$env:CONTAINER_SANDBOX_MOUNT_POINT") // join them and return with forward slashes so it can be serialized properly in json later if required - path := strings.TrimLeft(path, "c:") - path = strings.TrimLeft(path, "C:") path = filepath.Join(sandbox, path) return filepath.ToSlash(path) } diff --git a/node/pkg/cni/token_watch.go b/node/pkg/cni/token_watch.go index de0e788ab20..ae81a9827ea 100644 --- a/node/pkg/cni/token_watch.go +++ b/node/pkg/cni/token_watch.go @@ -64,7 +64,15 @@ func NamespaceOfUsedServiceAccount() string { } func BuildClientSet() (*kubernetes.Clientset, error) { - cfg, err := winutils.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")) + kubeconfig := os.Getenv("KUBECONFIG") + // Host env vars bleed through to the container on Windows HPC, so if cannot + // be trusted in this case + // FIXME: this will no longer be needed when containerd v1.6 is EOL'd + if winutils.InHostProcessContainer() { + kubeconfig = "" + } + cfg, err := winutils.BuildConfigFromFlags("", kubeconfig) + logrus.WithFields(logrus.Fields{"KUBECONFIG": kubeconfig, "cfg": cfg}).Debug("running cni.BuildClientSet") if err != nil { return nil, err } @@ -224,7 +232,14 @@ func Run() { for tu := range tokenChan { logrus.Info("Update of CNI kubeconfig triggered based on elapsed time.") - cfg, err := winutils.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")) + kubeconfig := os.Getenv("KUBECONFIG") + // Host env vars bleed through to the container on Windows HPC, so if cannot + // be trusted in this case + // FIXME: this will no longer be needed when containerd v1.6 is EOL'd + if winutils.InHostProcessContainer() { + kubeconfig = "" + } + cfg, err := winutils.BuildConfigFromFlags("", kubeconfig) if err != nil { logrus.WithError(err).Error("Error generating kube config.") continue diff --git a/node/pkg/lifecycle/shutdown/shutdown.go b/node/pkg/lifecycle/shutdown/shutdown.go index ba4499231da..2286218c746 100644 --- a/node/pkg/lifecycle/shutdown/shutdown.go +++ b/node/pkg/lifecycle/shutdown/shutdown.go @@ -43,7 +43,14 @@ func Run() { var clientset *kubernetes.Clientset // If running under kubernetes with secrets to call k8s API - if config, err := winutils.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")); err == nil { + kubeconfig := os.Getenv("KUBECONFIG") + // Host env vars bleed through to the container on Windows HPC, so if cannot + // be trusted in this case + // FIXME: this will no longer be needed when containerd v1.6 is EOL'd + if winutils.InHostProcessContainer() { + kubeconfig = "" + } + if config, err := winutils.BuildConfigFromFlags("", kubeconfig); err == nil { // default timeout is 30 seconds, which isn't appropriate for this kind of // shutdown action because network services, like kube-proxy might not be // running and we don't want to block the full 30 seconds if they are just diff --git a/node/pkg/lifecycle/startup/startup.go b/node/pkg/lifecycle/startup/startup.go index 452ac45556c..44046c142b6 100644 --- a/node/pkg/lifecycle/startup/startup.go +++ b/node/pkg/lifecycle/startup/startup.go @@ -130,7 +130,14 @@ func Run() { } // If running under kubernetes with secrets to call k8s API - if config, err := winutils.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")); err == nil { + kubeconfig := os.Getenv("KUBECONFIG") + // Host env vars bleed through to the container on Windows HPC, so if cannot + // be trusted in this case + // FIXME: this will no longer be needed when containerd v1.6 is EOL'd + if winutils.InHostProcessContainer() { + kubeconfig = "" + } + if config, err := winutils.BuildConfigFromFlags("", kubeconfig); err == nil { // default timeout is 30 seconds, which isn't appropriate for this kind of // startup action because network services, like kube-proxy might not be // running and we don't want to block the full 30 seconds if they are just @@ -330,7 +337,14 @@ func MonitorIPAddressSubnets() { if nodeRef := os.Getenv("CALICO_K8S_NODE_REF"); nodeRef != "" { k8sNodeName = nodeRef } - if config, err = winutils.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")); err == nil { + kubeconfig := os.Getenv("KUBECONFIG") + // Host env vars bleed through to the container on Windows HPC, so if cannot + // be trusted in this case + // FIXME: this will no longer be needed when containerd v1.6 is EOL'd + if winutils.InHostProcessContainer() { + kubeconfig = "" + } + if config, err = winutils.BuildConfigFromFlags("", kubeconfig); err == nil { // Create the k8s clientset. clientset, err = kubernetes.NewForConfig(config) if err != nil { diff --git a/node/windows-packaging/CalicoWindows/uninstall-calico-hpc.ps1 b/node/windows-packaging/CalicoWindows/uninstall-calico-hpc.ps1 index 219152fb6b8..d9d596438c0 100644 --- a/node/windows-packaging/CalicoWindows/uninstall-calico-hpc.ps1 +++ b/node/windows-packaging/CalicoWindows/uninstall-calico-hpc.ps1 @@ -87,5 +87,9 @@ Write-Host "Stopping and removing kube-proxy service if it is present..." Write-Host "It is recommended to run kube-proxy as kubernetes daemonset instead" Remove-CalicoService kube-proxy +Write-Host "Logging containerd CNI bin and conf dir paths:" +Get-Content "$env:ProgramFiles/containerd/config.toml" | Select-String -Pattern "^(\s)*bin_dir = (.)*$" +Get-Content "$env:ProgramFiles/containerd/config.toml" | Select-String -Pattern "^(\s)*conf_dir = (.)*$" + Get-Module 'calico' | Remove-Module -Force Write-Host "Done." From b67cd5a706e7008409ddd14211474fe5ec479005 Mon Sep 17 00:00:00 2001 From: Pedro Coutinho Date: Wed, 13 Sep 2023 18:31:59 -0700 Subject: [PATCH 15/16] Use same serviceaccount for windows and linux cni-plugin --- node/pkg/cni/token_watch.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/node/pkg/cni/token_watch.go b/node/pkg/cni/token_watch.go index ae81a9827ea..639fe056a92 100644 --- a/node/pkg/cni/token_watch.go +++ b/node/pkg/cni/token_watch.go @@ -7,7 +7,6 @@ import ( "fmt" "math/rand" "os" - "runtime" "strings" "sync" "time" @@ -24,7 +23,6 @@ import ( const ( defaultServiceAccountName = "calico-cni-plugin" - defaultServiceAccountNameWin = "calico-cni-plugin-windows" serviceAccountNamespace = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" tokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token" defaultCNITokenValiditySeconds = 24 * 60 * 60 @@ -260,9 +258,6 @@ func CNIServiceAccountName() string { logrus.WithField("name", sa).Debug("Using service account from CALICO_CNI_SERVICE_ACCOUNT") return sa } - if runtime.GOOS == "windows" { - return defaultServiceAccountNameWin - } return defaultServiceAccountName } From e90395cc7b90d2c67c6027c5b566078dfb1e8ecc Mon Sep 17 00:00:00 2001 From: Pedro Coutinho Date: Thu, 14 Sep 2023 20:17:14 -0700 Subject: [PATCH 16/16] Address review comments: - clarify HPC env var comments - add logging when skipping chmod on windows - fix kubeconfigPath prefix issue --- apiserver/cmd/apiserver/server/watch.go | 2 +- cni-plugin/pkg/install/install.go | 2 ++ cni-plugin/pkg/install/install_windows.go | 6 +++++- confd/pkg/backends/calico/routes.go | 2 +- confd/pkg/backends/calico/secret_watcher.go | 2 +- node/pkg/cni/token_watch.go | 4 ++-- node/pkg/lifecycle/shutdown/shutdown.go | 2 +- node/pkg/lifecycle/startup/startup.go | 4 ++-- 8 files changed, 15 insertions(+), 9 deletions(-) diff --git a/apiserver/cmd/apiserver/server/watch.go b/apiserver/cmd/apiserver/server/watch.go index 8825e4b25bf..338f30a6622 100644 --- a/apiserver/cmd/apiserver/server/watch.go +++ b/apiserver/cmd/apiserver/server/watch.go @@ -29,7 +29,7 @@ func WatchExtensionAuth(stopChan chan struct{}) (bool, error) { // set up k8s client // attempt 1: KUBECONFIG env var cfgFile := os.Getenv("KUBECONFIG") - // Host env vars bleed through to the container on Windows HPC, so if cannot + // Host env vars may override the container on Windows HPC, so $env:KUBECONFIG cannot // be trusted in this case // FIXME: this will no longer be needed when containerd v1.6 is EOL'd if winutils.InHostProcessContainer() { diff --git a/cni-plugin/pkg/install/install.go b/cni-plugin/pkg/install/install.go index 6542de7dd1c..5941ade1c88 100644 --- a/cni-plugin/pkg/install/install.go +++ b/cni-plugin/pkg/install/install.go @@ -440,6 +440,7 @@ func copyFileAndPermissions(src, dst string) (err error) { } if runtime.GOOS == "windows" { + logrus.Debug("chmod doesn't work on windows, skipping setting permissions") // chmod doesn't work on windows return } @@ -507,6 +508,7 @@ current-context: calico-context` func setSuidBit(file string) error { if runtime.GOOS == "windows" { // chmod doesn't work on windows + logrus.Debug("chmod doesn't work on windows, skipping setSuidBit()") return nil } fi, err := os.Stat(file) diff --git a/cni-plugin/pkg/install/install_windows.go b/cni-plugin/pkg/install/install_windows.go index 89be5ca4a72..6a884b6c24c 100644 --- a/cni-plugin/pkg/install/install_windows.go +++ b/cni-plugin/pkg/install/install_windows.go @@ -89,7 +89,11 @@ __KUBERNETES_ROUTE_POLICIES__ // Perform replacement of windows variables func replacePlatformSpecificVars(c config, netconf string) string { - kubeconfigPath := filepath.Join("c:", strings.TrimLeft(c.CNINetDir, "c:"), "/calico-kubeconfig") + cniNetDir := c.CNINetDir + if !(strings.HasPrefix(cniNetDir, "c:") || strings.HasPrefix(cniNetDir, "C:")) { + cniNetDir = filepath.Join("c:", cniNetDir) + } + kubeconfigPath := filepath.Join(cniNetDir, "/calico-kubeconfig") kubeconfigPath = filepath.ToSlash(kubeconfigPath) netconf = strings.Replace(netconf, "__KUBECONFIG_FILEPATH__", kubeconfigPath, -1) diff --git a/confd/pkg/backends/calico/routes.go b/confd/pkg/backends/calico/routes.go index e77583c5e6f..38903440e6b 100644 --- a/confd/pkg/backends/calico/routes.go +++ b/confd/pkg/backends/calico/routes.go @@ -72,7 +72,7 @@ func NewRouteGenerator(c *client) (rg *routeGenerator, err error) { // set up k8s client // attempt 1: KUBECONFIG env var cfgFile := os.Getenv("KUBECONFIG") - // Host env vars bleed through to the container on Windows HPC, so if cannot + // Host env vars may override the container on Windows HPC, so $env:KUBECONFIG cannot // be trusted in this case // FIXME: this will no longer be needed when containerd v1.6 is EOL'd if winutils.InHostProcessContainer() { diff --git a/confd/pkg/backends/calico/secret_watcher.go b/confd/pkg/backends/calico/secret_watcher.go index 9aeccdd0196..26eb38d1895 100644 --- a/confd/pkg/backends/calico/secret_watcher.go +++ b/confd/pkg/backends/calico/secret_watcher.go @@ -63,7 +63,7 @@ func NewSecretWatcher(c *client) (*secretWatcher, error) { // set up k8s client // attempt 1: KUBECONFIG env var cfgFile := os.Getenv("KUBECONFIG") - // Host env vars bleed through to the container on Windows HPC, so if cannot + // Host env vars may override the container on Windows HPC, so $env:KUBECONFIG cannot // be trusted in this case // FIXME: this will no longer be needed when containerd v1.6 is EOL'd if winutils.InHostProcessContainer() { diff --git a/node/pkg/cni/token_watch.go b/node/pkg/cni/token_watch.go index 639fe056a92..c2f14dbe52b 100644 --- a/node/pkg/cni/token_watch.go +++ b/node/pkg/cni/token_watch.go @@ -63,7 +63,7 @@ func NamespaceOfUsedServiceAccount() string { func BuildClientSet() (*kubernetes.Clientset, error) { kubeconfig := os.Getenv("KUBECONFIG") - // Host env vars bleed through to the container on Windows HPC, so if cannot + // Host env vars may override the container on Windows HPC, so $env:KUBECONFIG cannot // be trusted in this case // FIXME: this will no longer be needed when containerd v1.6 is EOL'd if winutils.InHostProcessContainer() { @@ -231,7 +231,7 @@ func Run() { for tu := range tokenChan { logrus.Info("Update of CNI kubeconfig triggered based on elapsed time.") kubeconfig := os.Getenv("KUBECONFIG") - // Host env vars bleed through to the container on Windows HPC, so if cannot + // Host env vars may override the container on Windows HPC, so $env:KUBECONFIG cannot // be trusted in this case // FIXME: this will no longer be needed when containerd v1.6 is EOL'd if winutils.InHostProcessContainer() { diff --git a/node/pkg/lifecycle/shutdown/shutdown.go b/node/pkg/lifecycle/shutdown/shutdown.go index 2286218c746..5cd80de06e5 100644 --- a/node/pkg/lifecycle/shutdown/shutdown.go +++ b/node/pkg/lifecycle/shutdown/shutdown.go @@ -44,7 +44,7 @@ func Run() { // If running under kubernetes with secrets to call k8s API kubeconfig := os.Getenv("KUBECONFIG") - // Host env vars bleed through to the container on Windows HPC, so if cannot + // Host env vars may override the container on Windows HPC, so $env:KUBECONFIG cannot // be trusted in this case // FIXME: this will no longer be needed when containerd v1.6 is EOL'd if winutils.InHostProcessContainer() { diff --git a/node/pkg/lifecycle/startup/startup.go b/node/pkg/lifecycle/startup/startup.go index 44046c142b6..cd8149033d5 100644 --- a/node/pkg/lifecycle/startup/startup.go +++ b/node/pkg/lifecycle/startup/startup.go @@ -131,7 +131,7 @@ func Run() { // If running under kubernetes with secrets to call k8s API kubeconfig := os.Getenv("KUBECONFIG") - // Host env vars bleed through to the container on Windows HPC, so if cannot + // Host env vars may override the container on Windows HPC, so $env:KUBECONFIG cannot // be trusted in this case // FIXME: this will no longer be needed when containerd v1.6 is EOL'd if winutils.InHostProcessContainer() { @@ -338,7 +338,7 @@ func MonitorIPAddressSubnets() { k8sNodeName = nodeRef } kubeconfig := os.Getenv("KUBECONFIG") - // Host env vars bleed through to the container on Windows HPC, so if cannot + // Host env vars may override the container on Windows HPC, so $env:KUBECONFIG cannot // be trusted in this case // FIXME: this will no longer be needed when containerd v1.6 is EOL'd if winutils.InHostProcessContainer() {