diff --git a/.gitignore b/.gitignore index 0828379c129..65a58edd155 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ crypto/tmp hack/release/release node/windows-packaging/CalicoWindows/confd/config-bgp.ps1 node/windows-packaging/CalicoWindows/confd/config-bgp.psm1 +node/windows-packaging/nssm.zip _output builder.coverprofile hack/release/ghr diff --git a/apiserver/cmd/apiserver/server/watch.go b/apiserver/cmd/apiserver/server/watch.go index 1a13fdb9eea..338f30a6622 100644 --- a/apiserver/cmd/apiserver/server/watch.go +++ b/apiserver/cmd/apiserver/server/watch.go @@ -7,12 +7,11 @@ import ( "os" "time" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" - "k8s.io/client-go/tools/clientcmd" ) const ( @@ -30,10 +29,16 @@ func WatchExtensionAuth(stopChan chan struct{}) (bool, error) { // set up k8s client // attempt 1: KUBECONFIG env var cfgFile := os.Getenv("KUBECONFIG") - cfg, err := clientcmd.BuildConfigFromFlags("", cfgFile) + // Host env vars may override the container on Windows HPC, so $env:KUBECONFIG cannot + // be trusted in this case + // FIXME: this will no longer be needed when containerd v1.6 is EOL'd + if winutils.InHostProcessContainer() { + cfgFile = "" + } + cfg, err := winutils.BuildConfigFromFlags("", cfgFile) if err != nil { // attempt 2: in cluster config - if cfg, err = rest.InClusterConfig(); err != nil { + if cfg, err = winutils.GetInClusterConfig(); err != nil { return false, err } } diff --git a/cni-plugin/Dockerfile-windows b/cni-plugin/Dockerfile-windows new file mode 100644 index 00000000000..745a77b1ad2 --- /dev/null +++ b/cni-plugin/Dockerfile-windows @@ -0,0 +1,39 @@ +# Copyright (c) 2023 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# FIXME: Use WINDOWS_HPC_VERSION and image instead of nanoserver and WINDOWS_VERSIONS when containerd v1.6 is EOL'd +# ARG WINDOWS_HPC_VERSION +ARG WINDOWS_VERSION + +# FROM mcr.microsoft.com/oss/kubernetes/windows-host-process-containers-base-image:${WINDOWS_HPC_VERSION} +FROM mcr.microsoft.com/windows/nanoserver:${WINDOWS_VERSION} + +ARG GIT_VERSION=unknown + +LABEL name="Calico Networking for CNI" \ + vendor="Project Calico" \ + version=$GIT_VERSION \ + release="1" \ + summary="Calico Networking for CNI" \ + description="Calico Networking for CNI includes a CNI networking plugin and CNI IPAM plugin" \ + maintainer="maintainers@projectcalico.org" + +ADD licenses/ /licenses +ADD LICENSE /licenses/ + +ADD bin/windows/ /opt/cni/bin/ + +ENV PATH=$PATH;/opt/cni/bin +WORKDIR /opt/cni/bin +CMD ["/opt/cni/bin/install.exe"] diff --git a/cni-plugin/Makefile b/cni-plugin/Makefile index e8a4c03f5b8..f870d01cda9 100644 --- a/cni-plugin/Makefile +++ b/cni-plugin/Makefile @@ -8,6 +8,7 @@ LOCAL_CHECKS=check-boring-ssl # Name of the images. # e.g., /: CNI_PLUGIN_IMAGE ?=cni +CNI_PLUGIN_IMAGE_WINDOWS ?=cni-windows BUILD_IMAGES ?=$(CNI_PLUGIN_IMAGE) ############################################################################### @@ -55,23 +56,28 @@ BIN=bin/$(ARCH) DEPLOY_CONTAINER_MARKER=$(DEPLOY_CONTAINER_STATIC_MARKER) endif -.PHONY: clean -clean: +.PHONY: clean clean-windows +clean: clean-windows # Clean .created files which indicate images / releases have been built. find . -name '.*.created*' -type f -delete find . -name '.*.published*' -type f -delete - rm -rf $(BIN) bin $(DEPLOY_CONTAINER_MARKER) .go-pkg-cache pkg/install/install.test + rm -rf $(BIN) bin $(DEPLOY_CONTAINER_MARKER) .go-pkg-cache pkg/install/install.test rm -f *.created rm -rf config/ + rm -rf dist/ + +clean-windows: clean-windows-builder + rm -rf $(WINDOWS_BIN) $(WINDOWS_DIST) ############################################################################### # Building the binary ############################################################################### +.PHONY: build build: $(BIN)/install $(BIN)/calico $(BIN)/calico-ipam ifeq ($(ARCH),amd64) # Go only supports amd64 for Windows builds. -BIN_WIN=bin/windows -build: $(BIN_WIN)/calico.exe $(BIN_WIN)/calico-ipam.exe +WINDOWS_BIN=bin/windows +build: $(WINDOWS_BIN)/install.exe $(WINDOWS_BIN)/calico.exe $(WINDOWS_BIN)/calico-ipam.exe endif # If ARCH is arm based, find the requested version/variant ifeq ($(word 1,$(subst v, ,$(ARCH))),arm) @@ -97,20 +103,23 @@ else endif ## Build the Calico network plugin and ipam plugins for Windows -$(BIN_WIN)/calico.exe $(BIN_WIN)/calico-ipam.exe: $(SRC_FILES) +$(WINDOWS_BIN)/install.exe: $(SRC_FILES) + -mkdir -p $(WINDOWS_BIN) $(DOCKER_RUN) \ -e GOOS=windows \ + -e GOARCH=amd64 \ -e CGO_ENABLED=1 \ - $(CALICO_BUILD) sh -c '$(GIT_CONFIG_SSH) \ - go build -v -buildvcs=false -o $(BIN_WIN)/calico.exe -ldflags "$(LDFLAGS)" $(PACKAGE_NAME)/cmd/calico && \ - go build -v -buildvcs=false -o $(BIN_WIN)/calico-ipam.exe -ldflags "$(LDFLAGS)" $(PACKAGE_NAME)/cmd/calico' + $(CALICO_BUILD) sh -c '$(GIT_CONFIG_SSH) \ + go build -v -buildvcs=false -o $(WINDOWS_BIN)/install.exe -ldflags "$(LDFLAGS)" $(PACKAGE_NAME)/cmd/calico' +$(WINDOWS_BIN)/calico-ipam.exe $(WINDOWS_BIN)/calico.exe: $(WINDOWS_BIN)/install.exe + cp "$<" "$@" ############################################################################### # Building the image ############################################################################### image: $(DEPLOY_CONTAINER_MARKER) -image-all: $(addprefix sub-image-,$(VALIDARCHES)) sub-image-fips-amd64 +image-all: $(addprefix sub-image-,$(VALIDARCHES)) sub-image-fips-amd64 image-windows sub-image-%: $(MAKE) image ARCH=$* sub-image-fips-%: @@ -128,17 +137,23 @@ $(DEPLOY_CONTAINER_FIPS_MARKER): Dockerfile.$(ARCH) build fetch-cni-bins $(MAKE) retag-build-images-with-registries VALIDARCHES=$(ARCH) IMAGETAG=latest-fips LATEST_IMAGE_TAG=latest-fips touch $@ -.PHONY: fetch-cni-bins +.PHONY: fetch-cni-bins fetch-win-cni-bins fetch-cni-bins: $(BIN)/flannel $(BIN)/loopback $(BIN)/host-local $(BIN)/portmap $(BIN)/tuning $(BIN)/bandwidth +fetch-win-cni-bins: $(WINDOWS_BIN)/flannel.exe $(BIN)/loopback $(BIN)/host-local $(BIN)/portmap $(BIN)/tuning $(BIN)/bandwidth: - mkdir -p $(BIN) + -mkdir -p $(BIN) $(CURL) -L --retry 5 $(CNI_ARTIFACTS_URL)/$(CNI_VERSION)/cni-plugins-linux-$(subst v7,,$(ARCH))-$(CNI_VERSION).tgz | tar -xz -C $(BIN) ./loopback ./host-local ./portmap ./tuning ./bandwidth $(BIN)/flannel: - mkdir -p $(BIN) + -mkdir -p $(BIN) $(CURL) -L --retry 5 https://github.com/flannel-io/cni-plugin/releases/download/$(FLANNEL_VERSION)/cni-plugin-flannel-linux-$(subst v7,,$(ARCH))-$(FLANNEL_VERSION).tgz | tar -xz -C $(BIN) ./flannel-$(subst v7,,$(ARCH)) - mv $(BIN)/flannel-$(subst v7,,$(ARCH)) $(BIN)/flannel + mv $(BIN)/flannel-$(subst v7,,$(ARCH)) $@ + +$(WINDOWS_BIN)/flannel.exe: + -mkdir -p $(WINDOWS_BIN) + $(CURL) -L --retry 5 https://github.com/flannel-io/cni-plugin/releases/download/$(FLANNEL_VERSION)/cni-plugin-flannel-windows-amd64-$(FLANNEL_VERSION).tgz | tar -xz -C $(WINDOWS_BIN) ./flannel-amd64.exe + mv $(WINDOWS_BIN)/flannel-amd64.exe $@ ############################################################################### # Unit Tests @@ -223,8 +238,8 @@ ci: clean mod-download build static-checks test-cni-versions image-all test-inst cd: image-all cd-common ## Build fv binary for Windows -$(BIN_WIN)/win-fv.exe: $(WINFV_SRCFILES) - $(DOCKER_RUN) -e GOOS=windows $(CALICO_BUILD) sh -c '$(GIT_CONFIG_SSH) go test ./win_tests -c -o $(BIN_WIN)/win-fv.exe' +$(WINDOWS_BIN)/win-fv.exe: $(WINFV_SRCFILES) + $(DOCKER_RUN) -e GOOS=windows $(CALICO_BUILD) sh -c '$(GIT_CONFIG_SSH) go test ./win_tests -c -o $(WINDOWS_BIN)/win-fv.exe' ############################################################################### # Release @@ -278,6 +293,75 @@ release-publish-latest: release-prereqs $(MAKE) push-images-to-registries push-manifests RELEASE=true IMAGETAG=latest RELEASE=$(RELEASE) CONFIRM=$(CONFIRM) +# FIXME: Use WINDOWS_HPC_VERSION and image instead of nanoserver and WINDOWS_VERSIONS when containerd v1.6 is EOL'd +# .PHONY: image-windows release-windows +# # Build Windows image and possibly push it to $DEV_REGISTRIES +# image-windows: setup-windows-builder Dockerfile-windows build fetch-win-cni-bins +# push="$${PUSH:-false}"; \ +# image_version="$${VERSION:-latest}"; \ +# for registry in $(DEV_REGISTRIES); do \ +# echo Building and pushing Windows image to $${registry}; \ +# image="$${registry}/$(CNI_PLUGIN_IMAGE_WINDOWS):$${image_version}"; \ +# docker buildx build \ +# --platform windows/amd64 \ +# --output=type=image,push=$${push} \ +# -t $${image} \ +# --pull \ +# --no-cache \ +# --build-arg GIT_VERSION=$(GIT_VERSION) \ +# --build-arg WINDOWS_HPC_VERSION=$(WINDOWS_HPC_VERSION) \ +# -f Dockerfile-windows .; \ +# done ; + +# # Build and push Windows image +# release-windows: release-prereqs clean-windows +# $(MAKE) image-windows PUSH=true + +WINDOWS_DIST = dist/windows + +$(WINDOWS_DIST)/$(CNI_PLUGIN_IMAGE_WINDOWS)-$(GIT_VERSION)-%.tar: sub-image-windows-$* + +sub-image-windows-%: setup-windows-builder Dockerfile-windows build fetch-win-cni-bins + # ensure dir for windows image tars exits + -mkdir -p $(WINDOWS_DIST) + docker buildx build \ + --platform windows/amd64 \ + --output=type=docker,dest=$(CURDIR)/$(WINDOWS_DIST)/$(CNI_PLUGIN_IMAGE_WINDOWS)-$(GIT_VERSION)-$*.tar \ + --pull \ + -t $(CNI_PLUGIN_IMAGE_WINDOWS):latest \ + --build-arg GIT_VERSION=$(GIT_VERSION) \ + --build-arg=WINDOWS_VERSION=$* \ + -f Dockerfile-windows . + +.PHONY: image-windows release-windows +image-windows: setup-windows-builder + for version in $(WINDOWS_VERSIONS); do \ + $(MAKE) sub-image-windows-$${version}; \ + done; + +release-windows: clean-windows image-windows + image_version="$${VERSION:-latest}"; \ + for registry in $(DEV_REGISTRIES); do \ + echo Pushing Windows images to $${registry}; \ + all_images=""; \ + manifest_image="$${registry}/$(CNI_PLUGIN_IMAGE_WINDOWS):$${image_version}"; \ + for win_ver in $(WINDOWS_VERSIONS); do \ + image_tar="$(WINDOWS_DIST)/$(CNI_PLUGIN_IMAGE_WINDOWS)-$(GIT_VERSION)-$${win_ver}.tar"; \ + image="$${registry}/$(CNI_PLUGIN_IMAGE_WINDOWS):$${image_version}-windows-$${win_ver}"; \ + echo Pushing image $${image} ...; \ + $(CRANE_BINDMOUNT) push $${image_tar} $${image}$(double_quote) & \ + all_images="$${all_images} $${image}"; \ + done; \ + wait; \ + $(DOCKER_MANIFEST) create --amend $${manifest_image} $${all_images}; \ + for win_ver in $(WINDOWS_VERSIONS); do \ + version=$$(docker manifest inspect mcr.microsoft.com/windows/nanoserver:$${win_ver} | grep "os.version" | head -n 1 | awk -F\" '{print $$4}'); \ + image="$${registry}/$(CNI_PLUGIN_IMAGE_WINDOWS):$${image_version}-windows-$${win_ver}"; \ + $(DOCKER_MANIFEST) annotate --os windows --arch amd64 --os-version $${version} $${manifest_image} $${image}; \ + done; \ + $(DOCKER_MANIFEST) push --purge $${manifest_image}; \ + done ; + ############################################################################### # Developer helper scripts (not used by build or test) ############################################################################### diff --git a/cni-plugin/cmd/calico/calico.go b/cni-plugin/cmd/calico/calico.go index 091c080fa51..055c9f26882 100644 --- a/cni-plugin/cmd/calico/calico.go +++ b/cni-plugin/cmd/calico/calico.go @@ -36,7 +36,7 @@ func main() { plugin.Main(VERSION) case "calico-ipam", "calico-ipam.exe": ipamplugin.Main(VERSION) - case "install": + case "install", "install.exe": err := install.Install() if err != nil { logrus.WithError(err).Fatal("Error installing CNI plugin") diff --git a/cni-plugin/pkg/install/install.go b/cni-plugin/pkg/install/install.go index 31fe35a6cc3..5941ade1c88 100644 --- a/cni-plugin/pkg/install/install.go +++ b/cni-plugin/pkg/install/install.go @@ -22,6 +22,7 @@ import ( "io" "os" "os/exec" + "runtime" "strings" "time" @@ -36,6 +37,7 @@ import ( "github.com/projectcalico/calico/libcalico-go/lib/logutils" "github.com/projectcalico/calico/libcalico-go/lib/names" "github.com/projectcalico/calico/libcalico-go/lib/seedrng" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" "github.com/projectcalico/calico/node/pkg/cni" ) @@ -126,7 +128,7 @@ func Install() error { logrus.SetFormatter(&logutils.Formatter{Component: "cni-installer"}) // Clean up any existing binaries / config / assets. - if err := os.RemoveAll("/host/etc/cni/net.d/calico-tls"); err != nil && !os.IsNotExist(err) { + if err := os.RemoveAll(winutils.GetHostPath("/host/etc/cni/net.d/calico-tls")); err != nil && !os.IsNotExist(err) { logrus.WithError(err).Warnf("Error removing old TLS directory") } @@ -136,12 +138,13 @@ func Install() error { // Determine if we're running as a Kubernetes pod. var kubecfg *rest.Config - serviceAccountTokenFile := "/var/run/secrets/kubernetes.io/serviceaccount/token" + serviceAccountTokenFile := winutils.GetHostPath("/var/run/secrets/kubernetes.io/serviceaccount/token") c.ServiceAccountToken = make([]byte, 0) var err error if fileExists(serviceAccountTokenFile) { logrus.Info("Running as a Kubernetes pod") - kubecfg, err = rest.InClusterConfig() + // FIXME: get rid of this and call rest.InClusterConfig() directly when containerd v1.6 is EOL'd + kubecfg, err = winutils.GetInClusterConfig() if err != nil { return err } @@ -160,14 +163,14 @@ func Install() error { // First check if the dir exists and has anything in it. if directoryExists(c.TLSAssetsDir) { logrus.Info("Installing any TLS assets") - mkdir("/host/etc/cni/net.d/calico-tls") - if err := copyFileAndPermissions(fmt.Sprintf("%s/%s", c.TLSAssetsDir, "etcd-ca"), "/host/etc/cni/net.d/calico-tls/etcd-ca"); err != nil { + mkdir(winutils.GetHostPath("/host/etc/cni/net.d/calico-tls")) + if err := copyFileAndPermissions(fmt.Sprintf("%s/%s", c.TLSAssetsDir, "etcd-ca"), winutils.GetHostPath("/host/etc/cni/net.d/calico-tls/etcd-ca")); err != nil { logrus.Warnf("Missing etcd-ca") } - if err := copyFileAndPermissions(fmt.Sprintf("%s/%s", c.TLSAssetsDir, "etcd-cert"), "/host/etc/cni/net.d/calico-tls/etcd-cert"); err != nil { + if err := copyFileAndPermissions(fmt.Sprintf("%s/%s", c.TLSAssetsDir, "etcd-cert"), winutils.GetHostPath("/host/etc/cni/net.d/calico-tls/etcd-cert")); err != nil { logrus.Warnf("Missing etcd-cert") } - if err := copyFileAndPermissions(fmt.Sprintf("%s/%s", c.TLSAssetsDir, "etcd-key"), "/host/etc/cni/net.d/calico-tls/etcd-key"); err != nil { + if err := copyFileAndPermissions(fmt.Sprintf("%s/%s", c.TLSAssetsDir, "etcd-key"), winutils.GetHostPath("/host/etc/cni/net.d/calico-tls/etcd-key")); err != nil { logrus.Warnf("Missing etcd-key") } } @@ -188,7 +191,7 @@ func Install() error { } // Place the new binaries if the directory is writeable. - dirs := []string{"/host/opt/cni/bin", "/host/secondary-bin-dir"} + dirs := []string{winutils.GetHostPath("/host/opt/cni/bin"), winutils.GetHostPath("/host/secondary-bin-dir")} binsWritten := false for _, d := range dirs { if err := fileutil.IsDirWriteable(d); err != nil { @@ -196,14 +199,20 @@ func Install() error { continue } + containerBinDir := "/opt/cni/bin/" + // The binaries dir in the container needs to be prepended by the CONTAINER_SANDBOX_MOUNT_POINT env var on Windows Host Process Containers + // see https://kubernetes.io/docs/tasks/configure-pod-container/create-hostprocess-pod/#containerd-v1-7-and-greater + if runtime.GOOS == "windows" { + containerBinDir = os.Getenv("CONTAINER_SANDBOX_MOUNT_POINT") + "/" + containerBinDir + } // Iterate through each binary we might want to install. - files, err := os.ReadDir("/opt/cni/bin/") + files, err := os.ReadDir(containerBinDir) if err != nil { logrus.Fatal(err) } for _, binary := range files { target := fmt.Sprintf("%s/%s", d, binary.Name()) - source := fmt.Sprintf("/opt/cni/bin/%s", binary.Name()) + source := fmt.Sprintf("%s/%s", containerBinDir, binary.Name()) if c.skipBinary(binary.Name()) { continue } @@ -276,7 +285,7 @@ func Install() error { logrus.Warn(err) } for _, f := range files { - if err = copyFileAndPermissions(c.TLSAssetsDir+"/"+f.Name(), "/host/etc/cni/net.d/calico-tls/"+f.Name()); err != nil { + if err = copyFileAndPermissions(winutils.GetHostPath(c.TLSAssetsDir+"/"+f.Name()), winutils.GetHostPath("/host/etc/cni/net.d/calico-tls/"+f.Name())); err != nil { logrus.Warn(err) continue } @@ -305,28 +314,7 @@ func isValidJSON(s string) error { } func writeCNIConfig(c config) { - netconf := `{ - "name": "k8s-pod-network", - "cniVersion": "0.3.1", - "plugins": [ - { - "type": "calico", - "log_level": "__LOG_LEVEL__", - "log_file_path": "__LOG_FILE_PATH__", - "datastore_type": "__DATASTORE_TYPE__", - "nodename": "__KUBERNETES_NODE_NAME__", - "mtu": __CNI_MTU__, - "ipam": {"type": "calico-ipam"}, - "policy": {"type": "k8s"}, - "kubernetes": {"kubeconfig": "__KUBECONFIG_FILEPATH__"} - }, - { - "type": "portmap", - "snat": true, - "capabilities": {"portMappings": true} - } - ] -}` + netconf := defaultNetConf() // Pick the config template to use. This can either be through an env var, // or a file mounted into the container. @@ -346,11 +334,15 @@ func writeCNIConfig(c config) { kubeconfigPath := c.CNINetDir + "/calico-kubeconfig" - // Perform replacements of variables. nodename, err := names.Hostname() if err != nil { logrus.Fatal(err) } + + // Perform replacement of platform specific variables + netconf = replacePlatformSpecificVars(c, netconf) + + // Perform replacements of variables. netconf = strings.Replace(netconf, "__LOG_LEVEL__", getEnv("LOG_LEVEL", "info"), -1) netconf = strings.Replace(netconf, "__LOG_FILE_PATH__", getEnv("LOG_FILE_PATH", "/var/log/calico/cni/cni.log"), -1) netconf = strings.Replace(netconf, "__LOG_FILE_MAX_SIZE__", getEnv("LOG_FILE_MAX_SIZE", "100"), -1) @@ -368,21 +360,21 @@ func writeCNIConfig(c config) { // Replace etcd datastore variables. hostSecretsDir := c.CNINetDir + "/calico-tls" - if fileExists("/host/etc/cni/net.d/calico-tls/etcd-cert") { + if fileExists(winutils.GetHostPath("/host/etc/cni/net.d/calico-tls/etcd-cert")) { etcdCertFile := fmt.Sprintf("%s/etcd-cert", hostSecretsDir) netconf = strings.Replace(netconf, "__ETCD_CERT_FILE__", etcdCertFile, -1) } else { netconf = strings.Replace(netconf, "__ETCD_CERT_FILE__", "", -1) } - if fileExists("/host/etc/cni/net.d/calico-tls/etcd-ca") { + if fileExists(winutils.GetHostPath("/host/etc/cni/net.d/calico-tls/etcd-ca")) { etcdCACertFile := fmt.Sprintf("%s/etcd-ca", hostSecretsDir) netconf = strings.Replace(netconf, "__ETCD_CA_CERT_FILE__", etcdCACertFile, -1) } else { netconf = strings.Replace(netconf, "__ETCD_CA_CERT_FILE__", "", -1) } - if fileExists("/host/etc/cni/net.d/calico-tls/etcd-key") { + if fileExists(winutils.GetHostPath("/host/etc/cni/net.d/calico-tls/etcd-key")) { etcdKeyFile := fmt.Sprintf("%s/etcd-key", hostSecretsDir) netconf = strings.Replace(netconf, "__ETCD_KEY_FILE__", etcdKeyFile, -1) } else { @@ -398,7 +390,7 @@ func writeCNIConfig(c config) { // Write out the file. name := getEnv("CNI_CONF_NAME", "10-calico.conflist") - path := fmt.Sprintf("/host/etc/cni/net.d/%s", name) + path := winutils.GetHostPath(fmt.Sprintf("/host/etc/cni/net.d/%s", name)) err = os.WriteFile(path, []byte(netconf), 0644) if err != nil { logrus.Fatal(err) @@ -408,15 +400,15 @@ func writeCNIConfig(c config) { if err != nil { logrus.Fatal(err) } - logrus.Infof("Created /host/etc/cni/net.d/%s", name) + logrus.Infof("Created %s", winutils.GetHostPath(fmt.Sprintf("/host/etc/cni/net.d/%s", name))) text := string(content) fmt.Println(text) // Remove any old config file, if one exists. oldName := getEnv("CNI_OLD_CONF_NAME", "10-calico.conflist") if name != oldName { - logrus.Infof("Removing /host/etc/cni/net.d/%s", oldName) - if err := os.Remove(fmt.Sprintf("/host/etc/cni/net.d/%s", oldName)); err != nil { + logrus.Infof("Removing %s", winutils.GetHostPath(fmt.Sprintf("/host/etc/cni/net.d/%s", oldName))) + if err := os.Remove(winutils.GetHostPath(fmt.Sprintf("/host/etc/cni/net.d/%s", oldName))); err != nil { logrus.WithError(err).Warnf("Failed to remove %s", oldName) } } @@ -447,6 +439,12 @@ func copyFileAndPermissions(src, dst string) (err error) { return fmt.Errorf("failed to rename file: %s", err) } + if runtime.GOOS == "windows" { + logrus.Debug("chmod doesn't work on windows, skipping setting permissions") + // chmod doesn't work on windows + return + } + // chmod the dst file to match the original permissions. si, err := os.Stat(src) if err != nil { @@ -502,12 +500,17 @@ current-context: calico-context` data = strings.Replace(data, "__TLS_CFG__", ca, -1) } - if err := os.WriteFile("/host/etc/cni/net.d/calico-kubeconfig", []byte(data), 0600); err != nil { + if err := os.WriteFile(winutils.GetHostPath("/host/etc/cni/net.d/calico-kubeconfig"), []byte(data), 0600); err != nil { logrus.Fatal(err) } } func setSuidBit(file string) error { + if runtime.GOOS == "windows" { + // chmod doesn't work on windows + logrus.Debug("chmod doesn't work on windows, skipping setSuidBit()") + return nil + } fi, err := os.Stat(file) if err != nil { return fmt.Errorf("failed to stat file: %s", err) diff --git a/cni-plugin/pkg/install/install_linux.go b/cni-plugin/pkg/install/install_linux.go new file mode 100644 index 00000000000..97e55ccfae2 --- /dev/null +++ b/cni-plugin/pkg/install/install_linux.go @@ -0,0 +1,48 @@ +//go:build !windows + +// Copyright (c) 2020 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package install + +func defaultNetConf() string { + netconf := `{ + "name": "k8s-pod-network", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "calico", + "log_level": "__LOG_LEVEL__", + "log_file_path": "__LOG_FILE_PATH__", + "datastore_type": "__DATASTORE_TYPE__", + "nodename": "__KUBERNETES_NODE_NAME__", + "mtu": __CNI_MTU__, + "ipam": {"type": "calico-ipam"}, + "policy": {"type": "k8s"}, + "kubernetes": {"kubeconfig": "__KUBECONFIG_FILEPATH__"} + }, + { + "type": "portmap", + "snat": true, + "capabilities": {"portMappings": true} + } + ] +}` + return netconf +} + +// This function is currently a no-op for Linux +func replacePlatformSpecificVars(c config, netconf string) string { + return netconf +} diff --git a/cni-plugin/pkg/install/install_windows.go b/cni-plugin/pkg/install/install_windows.go new file mode 100644 index 00000000000..6a884b6c24c --- /dev/null +++ b/cni-plugin/pkg/install/install_windows.go @@ -0,0 +1,186 @@ +//go:build windows + +// Copyright (c) 2020 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package install + +import ( + "fmt" + "path/filepath" + "strconv" + "strings" + + "github.com/sirupsen/logrus" + + "github.com/projectcalico/calico/libcalico-go/lib/winutils" +) + +func defaultNetConf() string { + netconf := `{ + "name": "Calico", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "calico", + "name": "Calico", + "windows_use_single_network": true, + "mode": "__MODE__", + "vxlan_mac_prefix": "__MAC_PREFIX__", + "vxlan_vni": __VNI__, + "mtu": __CNI_MTU__, + "policy": { + "type": "k8s" + }, + "log_level": "__LOG_LEVEL__", + "log_file_path": "__LOG_FILE_PATH__", + "windows_loopback_DSR": "__DSR_SUPPORT__", + "capabilities": {"dns": true}, + "DNS": { + "Nameservers": [__KUBERNETES_DNS_SERVERS__], + "Search": [ + "svc.cluster.local" + ] + }, + "nodename": "__KUBERNETES_NODE_NAME__", + "nodename_file": "__NODENAME_FILE__", + "nodename_file_optional": true, + "datastore_type": "__DATASTORE_TYPE__", + + "etcd_endpoints": "__ETCD_ENDPOINTS__", + "etcd_key_file": "__ETCD_KEY_FILE__", + "etcd_cert_file": "__ETCD_CERT_FILE__", + "etcd_ca_cert_file": "__ETCD_CA_CERT_FILE__", + + "kubernetes": { + "kubeconfig": "__KUBECONFIG_FILEPATH__" + }, + "ipam": { + "type": "__IPAM_TYPE__", + "subnet": "usePodCidr" + }, + + "policies": [ + { + "Name": "EndpointPolicy", + "Value": { + "Type": "OutBoundNAT", + "ExceptionList": [__KUBERNETES_SERVICE_CIDRS__] + } + }, +__KUBERNETES_ROUTE_POLICIES__ + ] + } + ] +}` + return netconf +} + +// Perform replacement of windows variables +func replacePlatformSpecificVars(c config, netconf string) string { + cniNetDir := c.CNINetDir + if !(strings.HasPrefix(cniNetDir, "c:") || strings.HasPrefix(cniNetDir, "C:")) { + cniNetDir = filepath.Join("c:", cniNetDir) + } + kubeconfigPath := filepath.Join(cniNetDir, "/calico-kubeconfig") + kubeconfigPath = filepath.ToSlash(kubeconfigPath) + netconf = strings.Replace(netconf, "__KUBECONFIG_FILEPATH__", kubeconfigPath, -1) + + netconf = strings.Replace(netconf, "__LOG_FILE_PATH__", getEnv("LOG_FILE_PATH", "c:/var/log/calico/cni/cni.log"), -1) + + // Support multiple KUBERNETES_SERVICE_CIDRS + serviceCIDRIPs := getEnv("KUBERNETES_SERVICE_CIDRS", "10.96.0.10") + serviceCIDRIPList := []string{} + for _, ip := range strings.Split(serviceCIDRIPs, ",") { + serviceCIDRIPList = append(serviceCIDRIPList, fmt.Sprintf("\"%s\"", strings.TrimSpace(ip))) + } + quotedServiceCIDRIPs := strings.Join(serviceCIDRIPList, ",") + netconf = strings.Replace(netconf, "__KUBERNETES_SERVICE_CIDRS__", quotedServiceCIDRIPs, -1) + + routePolicyList := []string{} + for _, ip := range serviceCIDRIPList { + routePolicy := fmt.Sprintf(` { + "Name": "EndpointPolicy", + "Value": { + "Type": "__ROUTE_TYPE__", + "DestinationPrefix": %s, + "NeedEncap": true + } + }`, ip) + routePolicyList = append(routePolicyList, routePolicy) + } + routePolicyListStr := strings.Join(routePolicyList, ",\n") + netconf = strings.Replace(netconf, "__KUBERNETES_ROUTE_POLICIES__", routePolicyListStr, -1) + + // __ROUTE_TYPE__ substitution must be done after __KUBERNETES_ROUTE_POLICIES__ because the latter contains the former. + netconf = strings.Replace(netconf, "__ROUTE_TYPE__", getEnv("ROUTE_TYPE", "SDNROUTE"), -1) + + netconf = strings.Replace(netconf, "__VNI__", getEnv("VXLAN_VNI", "4096"), -1) + netconf = strings.Replace(netconf, "__MAC_PREFIX__", getEnv("MAC_PREFIX", "0E-2A"), -1) + + netconf = strings.Replace(netconf, "__NODENAME_FILE__", getEnv("CALICO_NODENAME_FILE", "c:/var/run/calico/nodename"), -1) + + dnsIPs := getEnv("KUBERNETES_DNS_SERVERS", "10.96.0.10") + dnsIPList := []string{} + for _, ip := range strings.Split(dnsIPs, ",") { + dnsIPList = append(dnsIPList, fmt.Sprintf("\"%s\"", strings.TrimSpace(ip))) + } + quotedDNSIPs := strings.Join(dnsIPList, ",") + netconf = strings.Replace(netconf, "__KUBERNETES_DNS_SERVERS__", quotedDNSIPs, -1) + + backend := getEnv("CALICO_NETWORKING_BACKEND", "vxlan") + if strings.ToLower(backend) == "bird" || strings.ToLower(backend) == "bgp" { + backend = "windows-bgp" + } + netconf = strings.Replace(netconf, "__MODE__", backend, -1) + + ipamType := getEnv("CNI_IPAM_TYPE", "calico-ipam") + if backend == "vxlan" && ipamType != "calico-ipam" { + logrus.Fatalf("Calico VXLAN requires IPAM type calico-ipam, not %s", ipamType) + + } + netconf = strings.Replace(netconf, "__IPAM_TYPE__", ipamType, -1) + + stdout, stderr, err := winutils.Powershell("Get-ComputerInfo | select WindowsVersion, OsBuildNumber, OsHardwareAbstractionLayer") + if err != nil { + logrus.Fatalf("Failed to interact with powershell\nerror: %s\nstderr: %s", err, stderr) + } + line := strings.Split(stdout, "\r\n")[3] + fields := strings.Fields(line) + winVer := fields[0] + winVerInt, err := strconv.Atoi(winVer) + if err != nil { + logrus.Fatalf("Error converting winVer to int\nerror: %s", err) + } + buildNum := fields[1] + buildNumInt, err := strconv.Atoi(buildNum) + if err != nil { + logrus.Fatalf("Error converting buildNum to int\nerror: %s", err) + } + hal := fields[2] + halVer := strings.Split(hal, ".")[3] + halVerInt, err := strconv.Atoi(halVer) + if err != nil { + logrus.Fatalf("Error converting halVer to int\nerror: %s", err) + } + supportsDSR := (winVerInt == 1809 && buildNumInt >= 17763 && halVerInt >= 1432) || (winVerInt >= 1903 && buildNumInt >= 18317) + // Remove the quotes when replacing with boolean values (the quotes are in so that the template is valid JSON even before replacing) + if supportsDSR { + netconf = strings.Replace(netconf, `"__DSR_SUPPORT__"`, "true", -1) + } else { + netconf = strings.Replace(netconf, `"__DSR_SUPPORT__"`, "false", -1) + } + + return netconf +} diff --git a/cni-plugin/pkg/k8s/k8s.go b/cni-plugin/pkg/k8s/k8s.go index 124f1950023..733267f552c 100644 --- a/cni-plugin/pkg/k8s/k8s.go +++ b/cni-plugin/pkg/k8s/k8s.go @@ -30,10 +30,12 @@ import ( "github.com/containernetworking/plugins/pkg/ipam" libipam "github.com/projectcalico/calico/libcalico-go/lib/ipam" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" libapi "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" @@ -868,9 +870,18 @@ func NewK8sClient(conf types.NetConf, logger *logrus.Entry) (*kubernetes.Clients } // Use the kubernetes client code to load the kubeconfig file and combine it with the overrides. - config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( - &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig}, - configOverrides).ClientConfig() + var config *rest.Config + var err error + if winutils.InHostProcessContainer() { + // ClientConfig() calls InClusterConfig() at some point, which doesn't work + // on Windows HPC. Use winutils.GetInClusterConfig() instead in this case. + // FIXME: this will no longer be needed when containerd v1.6 is EOL'd + config, err = winutils.GetInClusterConfig() + } else { + config, err = clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig}, + configOverrides).ClientConfig() + } if err != nil { return nil, err } diff --git a/cni-plugin/pkg/plugin/plugin.go b/cni-plugin/pkg/plugin/plugin.go index 222d09c24b8..f9f22125839 100644 --- a/cni-plugin/pkg/plugin/plugin.go +++ b/cni-plugin/pkg/plugin/plugin.go @@ -37,9 +37,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" "github.com/projectcalico/calico/libcalico-go/lib/seedrng" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" @@ -92,7 +92,7 @@ func testConnection() error { // If we have a kubeconfig, test connection to the APIServer if conf.Kubernetes.Kubeconfig != "" { - k8sconfig, err := clientcmd.BuildConfigFromFlags("", conf.Kubernetes.Kubeconfig) + k8sconfig, err := winutils.BuildConfigFromFlags("", conf.Kubernetes.Kubeconfig) if err != nil { return fmt.Errorf("error building K8s client config: %s", err) } diff --git a/confd/pkg/backends/calico/routes.go b/confd/pkg/backends/calico/routes.go index 2ad321b7b99..38903440e6b 100644 --- a/confd/pkg/backends/calico/routes.go +++ b/confd/pkg/backends/calico/routes.go @@ -25,11 +25,10 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" - "k8s.io/client-go/tools/clientcmd" "github.com/projectcalico/calico/confd/pkg/resource/template" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" ) const ( @@ -73,11 +72,17 @@ func NewRouteGenerator(c *client) (rg *routeGenerator, err error) { // set up k8s client // attempt 1: KUBECONFIG env var cfgFile := os.Getenv("KUBECONFIG") - cfg, err := clientcmd.BuildConfigFromFlags("", cfgFile) + // Host env vars may override the container on Windows HPC, so $env:KUBECONFIG cannot + // be trusted in this case + // FIXME: this will no longer be needed when containerd v1.6 is EOL'd + if winutils.InHostProcessContainer() { + cfgFile = "" + } + cfg, err := winutils.BuildConfigFromFlags("", cfgFile) if err != nil { log.WithError(err).Info("KUBECONFIG environment variable not found, attempting in-cluster") // attempt 2: in cluster config - if cfg, err = rest.InClusterConfig(); err != nil { + if cfg, err = winutils.GetInClusterConfig(); err != nil { return } } diff --git a/confd/pkg/backends/calico/secret_watcher.go b/confd/pkg/backends/calico/secret_watcher.go index 73b347eecf2..26eb38d1895 100644 --- a/confd/pkg/backends/calico/secret_watcher.go +++ b/confd/pkg/backends/calico/secret_watcher.go @@ -23,9 +23,9 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" - "k8s.io/client-go/tools/clientcmd" + + "github.com/projectcalico/calico/libcalico-go/lib/winutils" ) type secretWatchData struct { @@ -63,11 +63,17 @@ func NewSecretWatcher(c *client) (*secretWatcher, error) { // set up k8s client // attempt 1: KUBECONFIG env var cfgFile := os.Getenv("KUBECONFIG") - cfg, err := clientcmd.BuildConfigFromFlags("", cfgFile) + // Host env vars may override the container on Windows HPC, so $env:KUBECONFIG cannot + // be trusted in this case + // FIXME: this will no longer be needed when containerd v1.6 is EOL'd + if winutils.InHostProcessContainer() { + cfgFile = "" + } + cfg, err := winutils.BuildConfigFromFlags("", cfgFile) if err != nil { log.WithError(err).Info("KUBECONFIG environment variable not found, attempting in-cluster") // attempt 2: in cluster config - if cfg, err = rest.InClusterConfig(); err != nil { + if cfg, err = winutils.GetInClusterConfig(); err != nil { return nil, err } } diff --git a/felix/config/param_types.go b/felix/config/param_types.go index 11a936fd8de..a0cdfb1f2b8 100644 --- a/felix/config/param_types.go +++ b/felix/config/param_types.go @@ -39,6 +39,7 @@ import ( "github.com/projectcalico/calico/felix/idalloc" "github.com/projectcalico/calico/felix/stringutils" cnet "github.com/projectcalico/calico/libcalico-go/lib/net" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" ) const ( @@ -265,6 +266,11 @@ type FileParam struct { } func (p *FileParam) Parse(raw string) (interface{}, error) { + // Use GetHostPath to use/resolve the CONTAINER_SANDBOX_MOUNT_POINT env var + // if running on Windows HPC. + // FIXME: this will no longer be needed when containerd v1.6 is EOL'd + raw = winutils.GetHostPath(raw) + if p.Executable { // Special case: for executable files, we search our directory // and the system path. diff --git a/felix/daemon/daemon.go b/felix/daemon/daemon.go index 56a1e189fe8..83ec939340c 100644 --- a/felix/daemon/daemon.go +++ b/felix/daemon/daemon.go @@ -31,9 +31,9 @@ import ( "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" "github.com/projectcalico/calico/libcalico-go/lib/backend/syncersv1/dedupebuffer" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" "github.com/projectcalico/calico/libcalico-go/lib/seedrng" @@ -309,7 +309,7 @@ configRetry: } else { // Not using KDD, fall back on trying to get a Kubernetes client from the environment. log.Info("Not using Kubernetes datastore driver, trying to get a Kubernetes client...") - k8sconf, err := rest.InClusterConfig() + k8sconf, err := winutils.GetInClusterConfig() if err != nil { log.WithError(err).Info("Kubernetes in-cluster config not available. " + "Assuming we're not in a Kubernetes deployment.") diff --git a/kube-controllers/cmd/kube-controllers/main.go b/kube-controllers/cmd/kube-controllers/main.go index 8d3e30d72cb..b225ab1c027 100644 --- a/kube-controllers/cmd/kube-controllers/main.go +++ b/kube-controllers/cmd/kube-controllers/main.go @@ -28,6 +28,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/projectcalico/calico/libcalico-go/lib/seedrng" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" log "github.com/sirupsen/logrus" "go.etcd.io/etcd/client/pkg/v3/srv" @@ -37,7 +38,6 @@ import ( "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" - "k8s.io/client-go/tools/clientcmd" "k8s.io/klog/v2" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" @@ -359,7 +359,7 @@ func getClients(kubeconfig string) (*kubernetes.Clientset, client.Interface, err // Now build the Kubernetes client, we support in-cluster config and kubeconfig // as means of configuring the client. - k8sconfig, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + k8sconfig, err := winutils.BuildConfigFromFlags("", kubeconfig) if err != nil { return nil, nil, fmt.Errorf("failed to build kubernetes client config: %s", err) } diff --git a/lib.Makefile b/lib.Makefile index 06970b5a837..e5a63d4ee59 100644 --- a/lib.Makefile +++ b/lib.Makefile @@ -1198,6 +1198,7 @@ release-dev-image-arch-to-registry-%: $(CRANE) cp $(DEV_REGISTRY)/$(BUILD_IMAGE):$(DEV_TAG)-$* $(RELEASE_REGISTRY)/$(BUILD_IMAGE):$(RELEASE_TAG)-$*$(double_quote) # release-prereqs checks that the environment is configured properly to create a release. +.PHONY: release-prereqs release-prereqs: ifndef VERSION $(error VERSION is undefined - run using make release VERSION=vX.Y.Z) @@ -1384,3 +1385,37 @@ help: @echo "CALICO_BUILD: $(CALICO_BUILD)" @echo "-----------------------------------------------------------" +############################################################################### +# Common functions for building windows images. +############################################################################### + +CRANE_BINDMOUNT_CMD := \ + docker run --rm \ + --net=host \ + --init \ + --entrypoint /bin/sh \ + -e LOCAL_USER_ID=$(LOCAL_USER_ID) \ + -v $(CURDIR):/go/src/$(PACKAGE_NAME):rw \ + -v $(DOCKER_CONFIG):/root/.docker/config.json \ + -w /go/src/$(PACKAGE_NAME) \ + $(CALICO_BUILD) -c $(double_quote)crane + +DOCKER_MANIFEST_CMD := docker manifest + +ifdef CONFIRM +CRANE_BINDMOUNT = $(CRANE_BINDMOUNT_CMD) +DOCKER_MANIFEST = $(DOCKER_MANIFEST_CMD) +else +CRANE_BINDMOUNT = echo [DRY RUN] $(CRANE_BINDMOUNT_CMD) +DOCKER_MANIFEST = echo [DRY RUN] $(DOCKER_MANIFEST_CMD) +endif + +# Clean up the docker builder used to create Windows image tarballs. +.PHONY: clean-windows-builder +clean-windows-builder: + -docker buildx rm calico-windows-builder + +# Set up the docker builder used to create Windows image tarballs. +.PHONY: setup-windows-builder +setup-windows-builder: clean-windows-builder + docker buildx create --name=calico-windows-builder --use --platform windows/amd64 diff --git a/libcalico-go/lib/backend/k8s/k8s.go b/libcalico-go/lib/backend/k8s/k8s.go index 7f48ff33fb9..4712f01b5ec 100644 --- a/libcalico-go/lib/backend/k8s/k8s.go +++ b/libcalico-go/lib/backend/k8s/k8s.go @@ -36,6 +36,7 @@ import ( "github.com/projectcalico/calico/libcalico-go/lib/backend/model" cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" "github.com/projectcalico/calico/libcalico-go/lib/net" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -335,6 +336,11 @@ func CreateKubernetesClientset(ca *apiconfig.CalicoAPIConfigSpec) (*rest.Config, return nil, nil, resources.K8sErrorToCalico(err, nil) } config, err = clientConfig.ClientConfig() + } else if winutils.InHostProcessContainer() { + // ClientConfig() calls InClusterConfig() at some point, which doesn't work + // on Windows HPC. Use winutils.GetInClusterConfig() instead in this case. + // FIXME: this will no longer be needed when containerd v1.6 is EOL'd + config, err = winutils.GetInClusterConfig() } else { config, err = clientcmd.NewNonInteractiveDeferredLoadingClientConfig( &loadingRules, configOverrides).ClientConfig() diff --git a/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/k8s.go b/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/k8s.go index 394bdaed3f3..4848103d937 100644 --- a/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/k8s.go +++ b/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/k8s.go @@ -35,6 +35,7 @@ import ( "github.com/projectcalico/calico/libcalico-go/lib/errors" "github.com/projectcalico/calico/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/custom" "github.com/projectcalico/calico/libcalico-go/lib/upgrade/migrator/clients/v1/k8s/resources" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" ) type KubeClient struct { @@ -84,8 +85,17 @@ func NewKubeClient(kc *capi.KubeConfig) (*KubeClient, error) { // A kubeconfig file was provided. Use it to load a config, passing through // any overrides. - config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( - &loadingRules, configOverrides).ClientConfig() + var config *rest.Config + var err error + if winutils.InHostProcessContainer() { + // ClientConfig() calls InClusterConfig() at some point, which doesn't work + // on Windows HPC. Use winutils.GetInClusterConfig() instead in this case. + // FIXME: this will no longer be needed when containerd v1.6 is EOL'd + config, err = winutils.GetInClusterConfig() + } else { + config, err = clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + &loadingRules, configOverrides).ClientConfig() + } if err != nil { return nil, resources.K8sErrorToCalico(err, nil) } diff --git a/libcalico-go/lib/winutils/winutils.go b/libcalico-go/lib/winutils/winutils.go new file mode 100644 index 00000000000..d9c428253b1 --- /dev/null +++ b/libcalico-go/lib/winutils/winutils.go @@ -0,0 +1,150 @@ +// Copyright (c) 2023 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package winutils + +import ( + "bytes" + "net" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + + "github.com/sirupsen/logrus" + + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + certutil "k8s.io/client-go/util/cert" +) + +func Powershell(args ...string) (string, string, error) { + // Add default powershell to PATH + path := os.Getenv("PATH") + err := os.Setenv("PATH", path+";C:/Windows/System32/WindowsPowerShell/v1.0/") + if err != nil { + return "", "", err + } + + ps, err := exec.LookPath("powershell.exe") + if err != nil { + return "", "", err + } + + args = append([]string{"-NoProfile", "-NonInteractive"}, args...) + cmd := exec.Command(ps, args...) + + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err = cmd.Run() + if err != nil { + return "", "", err + } + + return stdout.String(), stderr.String(), err +} + +// InHostProcessContainer returns true if inside a Windows HostProcess container, by +// checking if OS is Windows and the $env:CONTAINER_SANDBOX_MOUNT_POINT env variable +// is set. +func InHostProcessContainer() bool { + if runtime.GOOS == "windows" && os.Getenv("CONTAINER_SANDBOX_MOUNT_POINT") != "" { + return true + } + return false +} + +// GetHostPath returns the mount paths for a container +// In the case of Windows HostProcess containers this prepends the CONTAINER_SANDBOX_MOUNT_POINT env variable +// for other operating systems or if the sandbox env variable is not set it returns the standard mount points +// see https://kubernetes.io/docs/tasks/configure-pod-container/create-hostprocess-pod/#volume-mounts +// FIXME: this will no longer be needed when containerd v1.6 is EOL'd +func GetHostPath(path string) string { + if InHostProcessContainer() { + sandbox := os.Getenv("CONTAINER_SANDBOX_MOUNT_POINT") + // Remove drive letter prefixs as the CONTAINER_SANDBOX_MOUNT_POINT env var will contain it + path := strings.TrimPrefix(path, "c:") + path = strings.TrimPrefix(path, "C:") + // Remove literal unresolved CONTAINER_SANDBOX_MOUNT_POINT env var + path = strings.TrimPrefix(path, "$env:CONTAINER_SANDBOX_MOUNT_POINT") + // join them and return with forward slashes so it can be serialized properly in json later if required + path = filepath.Join(sandbox, path) + return filepath.ToSlash(path) + } + return path +} + +// FIXME: get rid of this and call rest.InClusterConfig() directly when containerd v1.6 is EOL'd +// GetInClusterConfig returns a config object which uses the service account +// kubernetes gives to pods. It's intended for clients that expect to be +// running inside a pod running on kubernetes. It will return ErrNotInCluster +// if called from a process not running in a kubernetes environment. +// It is a copy of InClusterConfig() from k8s.io/client-go/rest but using +// winutils.GetHostPath() for the file paths, so that Windows hostprocess +// containers on containerd v1.6 can work with the in-cluster config. +func GetInClusterConfig() (*rest.Config, error) { + tokenFile := GetHostPath("/var/run/secrets/kubernetes.io/serviceaccount/token") + rootCAFile := GetHostPath("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt") + host, port := os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT") + if len(host) == 0 || len(port) == 0 { + return nil, rest.ErrNotInCluster + } + + token, err := os.ReadFile(tokenFile) + if err != nil { + return nil, err + } + + tlsClientConfig := rest.TLSClientConfig{} + + if _, err := certutil.NewPool(rootCAFile); err != nil { + logrus.Errorf("Expected to load root CA config from %s, but got err: %v", rootCAFile, err) + } else { + tlsClientConfig.CAFile = rootCAFile + } + + return &rest.Config{ + Host: "https://" + net.JoinHostPort(host, port), + TLSClientConfig: tlsClientConfig, + BearerToken: string(token), + BearerTokenFile: tokenFile, + }, nil +} + +// FIXME: get rid of this and call clientcmd.BuildConfigFromFlags() directly when containerd v1.6 is EOL'd +// BuildConfigFromFlags is a helper function that builds configs from a master +// url or a kubeconfig filepath. These are passed in as command line flags for cluster +// components. Warnings should reflect this usage. If neither masterUrl or kubeconfigPath +// are passed in we fallback to inClusterConfig. If inClusterConfig fails, we fallback +// to the default config. +// It is a copy of BuildConfigFromFlags() from k8s.io/client-go/tools/clientcmd but using +// GetInClusterConfig(), which uses winutils.GetHostPath() for the file paths, so that +// Windows hostprocess containers on containerd v1.6 can work with the in-cluster config. +func BuildConfigFromFlags(masterUrl, kubeconfigPath string) (*rest.Config, error) { + if kubeconfigPath == "" && masterUrl == "" { + logrus.Warning("Neither --kubeconfig nor --master was specified. Using the inClusterConfig. This might not work.") + kubeconfig, err := GetInClusterConfig() + if err == nil { + return kubeconfig, nil + } + logrus.Warning("error creating inClusterConfig, falling back to default config: ", err) + } + return clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfigPath}, + &clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: masterUrl}}).ClientConfig() +} diff --git a/metadata.mk b/metadata.mk index 00d75fbd127..d21d8f4fe66 100644 --- a/metadata.mk +++ b/metadata.mk @@ -30,7 +30,13 @@ BIRD_VERSION=v0.3.3-202-g7a77fb73 # as both CI/CD and the release tooling will override this to build publishable images. DEV_REGISTRIES ?= calico -# RELEASE_REGISTIRES configures the container images registries which are published to +# RELEASE_REGISTIRES configures the container images registries which are published to # as part of an official release. # This variable is unused. Registries for releases are defined in hack/release/pkg/builder/builder.go # RELEASE_REGISTRIES = quay.io/calico docker.io/calico gcr.io/projectcalico-org eu.gcr.io/projectcalico-org asia.gcr.io/projectcalico-org us.gcr.io/projectcalico-org + +# FIXME: Use WINDOWS_HPC_VERSION and remove WINDOWS_VERSIONS when containerd v1.6 is EOL'd +# The Windows HPC container version used as base for Calico Windows images +WINDOWS_HPC_VERSION ?= v1.0.0 +# The Windows versions used as base for Calico Windows images +WINDOWS_VERSIONS ?= 1809 ltsc2022 diff --git a/node/.dockerignore b/node/.dockerignore index 41e4e8c7b36..707ac0ea867 100644 --- a/node/.dockerignore +++ b/node/.dockerignore @@ -8,3 +8,11 @@ # - CentOS repository file and our license !centos.repo !LICENSE +# - Files used in the windows image +!windows-packaging/CalicoWindows/libs +!windows-packaging/CalicoWindows/node +!windows-packaging/CalicoWindows/felix +!windows-packaging/CalicoWindows/confd +!windows-packaging/CalicoWindows/config-hpc.ps1 +!windows-packaging/CalicoWindows/uninstall-calico-hpc.ps1 +!windows-packaging/nssm.exe diff --git a/node/Dockerfile-windows b/node/Dockerfile-windows new file mode 100644 index 00000000000..f3c154b0261 --- /dev/null +++ b/node/Dockerfile-windows @@ -0,0 +1,49 @@ +# Copyright (c) 2023 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# FIXME: Use WINDOWS_HPC_VERSION and image instead of nanoserver and WINDOWS_VERSIONS when containerd v1.6 is EOL'd +# ARG WINDOWS_HPC_VERSION +ARG WINDOWS_VERSION + +#FROM mcr.microsoft.com/oss/kubernetes/windows-host-process-containers-base-image:${WINDOWS_HPC_VERSION} +FROM mcr.microsoft.com/windows/nanoserver:${WINDOWS_VERSION} + +ARG GIT_VERSION=unknown + +# Required labels for certification +LABEL name="Calico node" \ + vendor="Project Calico" \ + version=$GIT_VERSION \ + release="1" \ + summary="Calico node handles networking and policy for Calico" \ + description="Calico node handles networking and policy for Calico" \ + maintainer="maintainers@projectcalico.org" + +ADD LICENSE /licenses/ + +ADD dist/bin/calico-node.exe /CalicoWindows/calico-node.exe +ADD windows-packaging/CalicoWindows/felix/felix-service.ps1 /CalicoWindows/felix-service.ps1 +ADD windows-packaging/CalicoWindows/node/node-service.ps1 /CalicoWindows/node-service.ps1 +ADD windows-packaging/CalicoWindows/confd /CalicoWindows/confd +ADD windows-packaging/CalicoWindows/config-hpc.ps1 /CalicoWindows/config.ps1 +ADD windows-packaging/CalicoWindows/libs/calico/calico.psm1 /CalicoWindows/libs/calico/calico.psm1 +ADD windows-packaging/CalicoWindows/libs/hns/hns.psm1 /CalicoWindows/libs/hns/hns.psm1 +ADD windows-packaging/nssm.exe /nssm.exe +ADD windows-packaging/CalicoWindows/uninstall-calico-hpc.ps1 /uninstall-calico.ps1 + +ENV PATH=$PATH;/CalicoWindows +WORKDIR /CalicoWindows +# The nanoserver image does not have powershell but this works because +# this container will be running on the host. +ENTRYPOINT ["powershell"] diff --git a/node/Makefile b/node/Makefile index 67f4d43ee44..1c54c10b4c4 100644 --- a/node/Makefile +++ b/node/Makefile @@ -8,7 +8,7 @@ DEV_TAG_SUFFIX ?=0.dev # Name of the images. # e.g., /: NODE_IMAGE ?=node -WINDOWS_IMAGE ?=windows +WINDOWS_IMAGE ?=node-windows WINDOWS_UPGRADE_IMAGE ?=windows-upgrade WINDOWS_VERSIONS?=1809 2004 20H2 ltsc2022 @@ -75,6 +75,7 @@ TEST_CONTAINER_FILES=$(shell find tests/ -type f ! -name '*.created') # Variables controlling the image NODE_CONTAINER_CREATED=.calico_node.created-$(ARCH) NODE_CONTAINER_FIPS_CREATED=.calico_node.created-$(ARCH)-fips +NODE_CONTAINER_WINDOWS_CREATED=.calico_node-windows.created WINDOWS_BINARY = $(NODE_CONTAINER_BIN_DIR)/calico-node.exe TOOLS_MOUNTNS_BINARY = $(NODE_CONTAINER_BIN_DIR)/mountns-$(ARCH) @@ -192,14 +193,16 @@ clean: clean-windows docker rmi $(NODE_IMAGE):latest-$(ARCH) || true docker rmi $(NODE_IMAGE):latest-fips-$(ARCH) || true docker rmi $(TEST_CONTAINER_NAME) || true + docker rmi $(addprefix $(WINDOWS_IMAGE):latest-,$(WINDOWS_VERSIONS)) || true docker rmi $(addprefix $(WINDOWS_UPGRADE_IMAGE):latest-,$(WINDOWS_VERSIONS)) || true -clean-windows: +clean-windows: clean-windows-builder -rm -f $(WINDOWS_ARCHIVE) $(WINDOWS_ARCHIVE_BINARY) $(WINDOWS_BINARY) -rm -f $(WINDOWS_ARCHIVE_ROOT)/libs/hns/hns.psm1 -rm -f $(WINDOWS_ARCHIVE_ROOT)/libs/hns/License.txt -rm -f $(WINDOWS_ARCHIVE_ROOT)/cni/*.exe -rm -f $(WINDOWS_ARCHIVE_ROOT)/../nssm.zip + -rm -f $(WINDOWS_ARCHIVE_ROOT)/../nssm.exe -rm -f $(WINDOWS_INSTALL_SCRIPT) -rm -rf "$(WINDOWS_DIST)" -rm -rf "$(WINDOWS_UPGRADE_DIST)" @@ -286,7 +289,7 @@ endif # Building the image ############################################################################### ## Create the images for all supported ARCHes -image-all: $(addprefix sub-image-,$(VALIDARCHES)) sub-image-fips-amd64 +image-all: $(addprefix sub-image-,$(VALIDARCHES)) sub-image-fips-amd64 image-windows sub-image-%: $(MAKE) image ARCH=$* sub-image-fips-%: @@ -475,10 +478,10 @@ st: $(REMOTE_DEPS) image dist/calicoctl busybox.tar calico-node.tar workload.tar # CI/CD ############################################################################### .PHONY: ci -ci: static-checks ut image build-windows-upgrade-archive image-tar-windows-all st +ci: static-checks ut image image-windows st ## Deploys images to registry -cd: cd-common cd-windows-all +cd: cd-common release-windows check-boring-ssl: $(NODE_CONTAINER_BIN_DIR)/calico-node-amd64 $(DOCKER_RUN) -e CGO_ENABLED=$(CGO_ENABLED) $(CALICO_BUILD) \ @@ -506,7 +509,7 @@ release-build: .release-$(VERSION).created touch $@ ## Produces the Windows installation ZIP archive for the release. -release-windows-archive $(WINDOWS_ARCHIVE): release-prereqs +release-windows-archive: release-prereqs $(MAKE) build-windows-archive WINDOWS_ARCHIVE_TAG=$(VERSION) ## Verifies the release artifacts produces by `make release-build` are correct. @@ -566,7 +569,17 @@ $(WINDOWS_ARCHIVE_ROOT)/libs/hns/License.txt: windows-packaging/nssm.zip: wget -O windows-packaging/nssm.zip https://nssm.cc/ci/nssm-$(WINDOWS_NSSM_VERSION).zip -build-windows-archive: $(WINDOWS_ARCHIVE_FILES) windows-packaging/nssm.zip +windows-packaging/nssm.exe: windows-packaging/nssm.zip + cd windows-packaging && \ + sha256sum --check nssm.sha256sum && \ + unzip -o nssm.zip 'nssm-$(WINDOWS_NSSM_VERSION)/win64/nssm.exe'&& \ + mv nssm-$(WINDOWS_NSSM_VERSION)/win64/nssm.exe nssm.exe && \ + rm -rf nssm-$(WINDOWS_NSSM_VERSION)/ + +$(WINDOWS_ARCHIVE): build-windows-archive + +.PHONY: build-windows-archive +build-windows-archive: build $(WINDOWS_ARCHIVE_FILES) windows-packaging/nssm.zip # To be as atomic as possible, we re-do work like unpacking NSSM here. -rm -f "$(WINDOWS_ARCHIVE)" -rm -rf $(WINDOWS_ARCHIVE_ROOT)/nssm @@ -624,37 +637,11 @@ build-windows-upgrade-archive: clean-windows $(WINDOWS_UPGRADE_INSTALL_ZIP) $(WI rm $(WINDOWS_UPGRADE_ARCHIVE) || true cd $(WINDOWS_UPGRADE_DIST_STAGE) && zip -r "$(CURDIR)/$(WINDOWS_UPGRADE_ARCHIVE)" *.zip *.ps1 -# Sets up the docker builder used to create Windows image tarballs. -setup-windows-builder: - -docker buildx rm calico-windows-builder - docker buildx create --name=calico-windows-builder --use --platform windows/amd64 - # Builds all the Windows image tarballs for each version in WINDOWS_VERSIONS image-tar-windows-all: setup-windows-builder $(addprefix sub-image-tar-windows-,$(WINDOWS_VERSIONS)) $(addprefix sub-image-tar-windows-upgrade-,$(WINDOWS_VERSIONS)) -CRANE_BINDMOUNT_CMD := \ - docker run --rm \ - --net=host \ - --init \ - --entrypoint /bin/sh \ - -e LOCAL_USER_ID=$(LOCAL_USER_ID) \ - -v $(CURDIR):/go/src/$(PACKAGE_NAME):rw \ - -v $(DOCKER_CONFIG):/root/.docker/config.json \ - -w /go/src/$(PACKAGE_NAME) \ - $(CALICO_BUILD) -c $(double_quote)crane - -DOCKER_MANIFEST_CMD := docker manifest - -ifdef CONFIRM -CRANE_BINDMOUNT = $(CRANE_BINDMOUNT_CMD) -DOCKER_MANIFEST = $(DOCKER_MANIFEST_CMD) -else -CRANE_BINDMOUNT = echo [DRY RUN] $(CRANE_BINDMOUNT_CMD) -DOCKER_MANIFEST = echo [DRY RUN] $(DOCKER_MANIFEST_CMD) -endif - # Uses the docker builder to create a Windows image tarball for a single Windows version. -sub-image-tar-windows-%: +sub-image-tar-windows-%: $(WINDOWS_ARCHIVE_BINARY) dist/calico-windows-$(GIT_VERSION).zip $(WINDOWS_INSTALL_SCRIPT) # ensure dir for windows image tars -mkdir -p $(WINDOWS_DIST) # ensure docker build dir exists @@ -733,3 +720,70 @@ endif done; \ $(DOCKER_MANIFEST) push --purge $${manifest_image}; \ done ; + +# FIXME: Use WINDOWS_HPC_VERSION and image instead of nanoserver and WINDOWS_VERSIONS when containerd v1.6 is EOL'd +# .PHONY: image-windows release-windows +# # Build Windows image and possibly push it to $DEV_REGISTRIES +# image-windows: setup-windows-builder Dockerfile-windows $(WINDOWS_BINARY) $(WINDOWS_ARCHIVE_ROOT)/libs/hns/hns.psm1 $(WINDOWS_ARCHIVE_ROOT)/libs/calico/calico.psm1 $(WINDOWS_ARCHIVE_ROOT)/config-hpc.ps1 $(WINDOWS_ARCHIVE_ROOT)/felix/felix-service.ps1 $(WINDOWS_ARCHIVE_ROOT)/node/node-service.ps1 $(WINDOWS_ARCHIVE_ROOT)/uninstall-calico-hpc.ps1 windows-packaging/nssm.exe +# push="$${PUSH:-false}"; \ +# image_version="$${VERSION:-latest}"; \ +# for registry in $(DEV_REGISTRIES); do \ +# echo Building and pushing Windows image to $${registry}; \ +# image="$${registry}/$(WINDOWS_IMAGE):$${image_version}"; \ +# docker buildx build \ +# --platform windows/amd64 \ +# --output=type=image,push=$${push} \ +# -t $${image} \ +# --pull \ +# --no-cache \ +# --build-arg GIT_VERSION=$(GIT_VERSION) \ +# --build-arg WINDOWS_HPC_VERSION=$(WINDOWS_HPC_VERSION) \ +# -f Dockerfile-windows .; \ +# done ; + +# # Build and push Windows image +# release-windows: release-prereqs clean-windows +# $(MAKE) image-windows PUSH=true + +$(WINDOWS_DIST)/$(WINDOWS_IMAGE)-$(GIT_VERSION)-%.tar: sub-image-windows-$* + +sub-image-windows-%: Dockerfile-windows $(WINDOWS_BINARY) $(WINDOWS_ARCHIVE_ROOT)/libs/hns/hns.psm1 $(WINDOWS_ARCHIVE_ROOT)/libs/calico/calico.psm1 $(WINDOWS_ARCHIVE_ROOT)/config-hpc.ps1 $(WINDOWS_ARCHIVE_ROOT)/felix/felix-service.ps1 $(WINDOWS_ARCHIVE_ROOT)/node/node-service.ps1 $(WINDOWS_ARCHIVE_ROOT)/uninstall-calico-hpc.ps1 windows-packaging/nssm.exe + # ensure dir for windows image tars exits + -mkdir -p $(WINDOWS_DIST) + docker buildx build \ + --platform windows/amd64 \ + --output=type=docker,dest=$(CURDIR)/$(WINDOWS_DIST)/$(WINDOWS_IMAGE)-$(GIT_VERSION)-$*.tar \ + --pull \ + -t $(WINDOWS_IMAGE):latest \ + --build-arg GIT_VERSION=$(GIT_VERSION) \ + --build-arg=WINDOWS_VERSION=$* \ + -f Dockerfile-windows . + +.PHONY: image-windows release-windows +image-windows: setup-windows-builder + for version in $(WINDOWS_VERSIONS); do \ + $(MAKE) sub-image-windows-$${version}; \ + done; + +release-windows: clean-windows image-windows + image_version="$${VERSION:-latest}"; \ + for registry in $(DEV_REGISTRIES); do \ + echo Pushing Windows images to $${registry}; \ + all_images=""; \ + manifest_image="$${registry}/$(WINDOWS_IMAGE):$${image_version}"; \ + for win_ver in $(WINDOWS_VERSIONS); do \ + image_tar="$(WINDOWS_DIST)/$(WINDOWS_IMAGE)-$(GIT_VERSION)-$${win_ver}.tar"; \ + image="$${registry}/$(WINDOWS_IMAGE):$${image_version}-windows-$${win_ver}"; \ + echo Pushing image $${image} ...; \ + $(CRANE_BINDMOUNT) push $${image_tar} $${image}$(double_quote) & \ + all_images="$${all_images} $${image}"; \ + done; \ + wait; \ + $(DOCKER_MANIFEST) create --amend $${manifest_image} $${all_images}; \ + for win_ver in $(WINDOWS_VERSIONS); do \ + version=$$(docker manifest inspect mcr.microsoft.com/windows/nanoserver:$${win_ver} | grep "os.version" | head -n 1 | awk -F\" '{print $$4}'); \ + image="$${registry}/$(WINDOWS_IMAGE):$${image_version}-windows-$${win_ver}"; \ + $(DOCKER_MANIFEST) annotate --os windows --arch amd64 --os-version $${version} $${manifest_image} $${image}; \ + done; \ + $(DOCKER_MANIFEST) push --purge $${manifest_image}; \ + done ; diff --git a/node/cmd/calico-node/main.go b/node/cmd/calico-node/main.go index 7138b21e2c5..7f644a1b1ad 100644 --- a/node/cmd/calico-node/main.go +++ b/node/cmd/calico-node/main.go @@ -38,7 +38,6 @@ import ( "github.com/projectcalico/calico/node/pkg/lifecycle/shutdown" "github.com/projectcalico/calico/node/pkg/lifecycle/startup" "github.com/projectcalico/calico/node/pkg/status" - "github.com/projectcalico/calico/node/pkg/winupgrade" ) // Create a new flag set. @@ -51,8 +50,6 @@ var runBPF = flagSet.Bool("bpf", false, "Run BPF debug tool") var runInit = flagSet.Bool("init", false, "Do privileged initialisation of a new node (mount file systems etc).") var bestEffort = flagSet.Bool("best-effort", false, "Used in combination with the init flag. Report errors but do not fail if an error occures during initialisation.") var runStartup = flagSet.Bool("startup", false, "Do non-privileged start-up routine.") -var runWinUpgrade = flagSet.Bool("upgrade-windows", false, "Run Windows upgrade service.") -var runShouldInstallWindowsUpgrade = flagSet.Bool("should-install-windows-upgrade", false, "Check if Windows upgrade service should be installed.") var runShutdown = flagSet.Bool("shutdown", false, "Do shutdown routine.") var monitorAddrs = flagSet.Bool("monitor-addresses", false, "Monitor change in node IP addresses") var runAllocateTunnelAddrs = flagSet.Bool("allocate-tunnel-addrs", false, "Configure tunnel addresses for this node") @@ -147,12 +144,6 @@ func main() { } else if *runShutdown { logrus.SetFormatter(&logutils.Formatter{Component: "shutdown"}) shutdown.Run() - } else if *runWinUpgrade { - logrus.SetFormatter(&logutils.Formatter{Component: "windows-upgrade"}) - winupgrade.Run() - } else if *runShouldInstallWindowsUpgrade { - logrus.SetFormatter(&logutils.Formatter{Component: "should-install-windows-upgrade"}) - winupgrade.ShouldInstallUpgradeService() } else if *monitorAddrs { logrus.SetFormatter(&logutils.Formatter{Component: "monitor-addresses"}) startup.ConfigureLogging() diff --git a/node/pkg/cni/token_watch.go b/node/pkg/cni/token_watch.go index ad60157480c..c2f14dbe52b 100644 --- a/node/pkg/cni/token_watch.go +++ b/node/pkg/cni/token_watch.go @@ -17,7 +17,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" + + "github.com/projectcalico/calico/libcalico-go/lib/winutils" ) const ( @@ -27,10 +28,9 @@ const ( defaultCNITokenValiditySeconds = 24 * 60 * 60 minTokenRetryDuration = 5 * time.Second defaultRefreshFraction = 4 + kubeconfigPath = "/host/etc/cni/net.d/calico-kubeconfig" ) -var kubeconfigPath string = "/host/etc/cni/net.d/calico-kubeconfig" - type TokenRefresher struct { tokenSupported bool tokenOnce *sync.Once @@ -54,7 +54,7 @@ type TokenUpdate struct { } func NamespaceOfUsedServiceAccount() string { - namespace, err := os.ReadFile(serviceAccountNamespace) + namespace, err := os.ReadFile(winutils.GetHostPath(serviceAccountNamespace)) if err != nil { logrus.WithError(err).Fatal("Failed to read service account namespace file") } @@ -62,7 +62,15 @@ func NamespaceOfUsedServiceAccount() string { } func BuildClientSet() (*kubernetes.Clientset, error) { - cfg, err := clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")) + kubeconfig := os.Getenv("KUBECONFIG") + // Host env vars may override the container on Windows HPC, so $env:KUBECONFIG cannot + // be trusted in this case + // FIXME: this will no longer be needed when containerd v1.6 is EOL'd + if winutils.InHostProcessContainer() { + kubeconfig = "" + } + cfg, err := winutils.BuildConfigFromFlags("", kubeconfig) + logrus.WithFields(logrus.Fields{"KUBECONFIG": kubeconfig, "cfg": cfg}).Debug("running cni.BuildClientSet") if err != nil { return nil, err } @@ -177,7 +185,7 @@ func (t *TokenRefresher) tokenRequestSupported(clientset *kubernetes.Clientset) } func tokenUpdateFromFile() (TokenUpdate, error) { - tokenBytes, err := os.ReadFile(tokenFile) + tokenBytes, err := os.ReadFile(winutils.GetHostPath(tokenFile)) if err != nil { logrus.WithError(err).Error("Failed to read service account token file") return TokenUpdate{}, err @@ -222,7 +230,14 @@ func Run() { for tu := range tokenChan { logrus.Info("Update of CNI kubeconfig triggered based on elapsed time.") - cfg, err := clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")) + kubeconfig := os.Getenv("KUBECONFIG") + // Host env vars may override the container on Windows HPC, so $env:KUBECONFIG cannot + // be trusted in this case + // FIXME: this will no longer be needed when containerd v1.6 is EOL'd + if winutils.InHostProcessContainer() { + kubeconfig = "" + } + cfg, err := winutils.BuildConfigFromFlags("", kubeconfig) if err != nil { logrus.WithError(err).Error("Error generating kube config.") continue @@ -237,7 +252,7 @@ func Run() { } // CNIServiceAccountName returns the name of the serviceaccount to use for the CNI plugin token request. -// This can be set via the CALICO_CNI_SERVICE_ACCOUNT environment variable, and defaults to "calico-cni-plugin" otherwise. +// This can be set via the CALICO_CNI_SERVICE_ACCOUNT environment variable, and defaults to "calico-cni-plugin" (on Linux, "calico-cni-plugin-windows" on Windows) otherwise. func CNIServiceAccountName() string { if sa := os.Getenv("CALICO_CNI_SERVICE_ACCOUNT"); sa != "" { logrus.WithField("name", sa).Debug("Using service account from CALICO_CNI_SERVICE_ACCOUNT") @@ -271,9 +286,9 @@ current-context: calico-context` data := fmt.Sprintf(template, cfg.Host, base64.StdEncoding.EncodeToString(cfg.CAData), token) // Write the filled out config to disk. - if err := os.WriteFile(kubeconfigPath, []byte(data), 0600); err != nil { + if err := os.WriteFile(winutils.GetHostPath(kubeconfigPath), []byte(data), 0600); err != nil { logrus.WithError(err).Error("Failed to write CNI plugin kubeconfig file") return } - logrus.WithField("path", kubeconfigPath).Info("Wrote updated CNI kubeconfig file.") + logrus.WithField("path", winutils.GetHostPath(kubeconfigPath)).Info("Wrote updated CNI kubeconfig file.") } diff --git a/node/pkg/lifecycle/shutdown/shutdown.go b/node/pkg/lifecycle/shutdown/shutdown.go index 21a3fc06cd7..5cd80de06e5 100644 --- a/node/pkg/lifecycle/shutdown/shutdown.go +++ b/node/pkg/lifecycle/shutdown/shutdown.go @@ -19,8 +19,8 @@ import ( log "github.com/sirupsen/logrus" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" "github.com/projectcalico/calico/node/pkg/lifecycle/utils" ) @@ -43,7 +43,14 @@ func Run() { var clientset *kubernetes.Clientset // If running under kubernetes with secrets to call k8s API - if config, err := clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")); err == nil { + kubeconfig := os.Getenv("KUBECONFIG") + // Host env vars may override the container on Windows HPC, so $env:KUBECONFIG cannot + // be trusted in this case + // FIXME: this will no longer be needed when containerd v1.6 is EOL'd + if winutils.InHostProcessContainer() { + kubeconfig = "" + } + if config, err := winutils.BuildConfigFromFlags("", kubeconfig); err == nil { // default timeout is 30 seconds, which isn't appropriate for this kind of // shutdown action because network services, like kube-proxy might not be // running and we don't want to block the full 30 seconds if they are just diff --git a/node/pkg/lifecycle/startup/startup.go b/node/pkg/lifecycle/startup/startup.go index b2e9e11ed87..cd8149033d5 100644 --- a/node/pkg/lifecycle/startup/startup.go +++ b/node/pkg/lifecycle/startup/startup.go @@ -31,7 +31,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/api/pkg/lib/numorstring" @@ -45,6 +44,7 @@ import ( "github.com/projectcalico/calico/libcalico-go/lib/selector" "github.com/projectcalico/calico/libcalico-go/lib/upgrade/migrator" "github.com/projectcalico/calico/libcalico-go/lib/upgrade/migrator/clients" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" "github.com/projectcalico/calico/node/pkg/calicoclient" "github.com/projectcalico/calico/node/pkg/lifecycle/startup/autodetection" @@ -130,7 +130,14 @@ func Run() { } // If running under kubernetes with secrets to call k8s API - if config, err := clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")); err == nil { + kubeconfig := os.Getenv("KUBECONFIG") + // Host env vars may override the container on Windows HPC, so $env:KUBECONFIG cannot + // be trusted in this case + // FIXME: this will no longer be needed when containerd v1.6 is EOL'd + if winutils.InHostProcessContainer() { + kubeconfig = "" + } + if config, err := winutils.BuildConfigFromFlags("", kubeconfig); err == nil { // default timeout is 30 seconds, which isn't appropriate for this kind of // startup action because network services, like kube-proxy might not be // running and we don't want to block the full 30 seconds if they are just @@ -330,7 +337,14 @@ func MonitorIPAddressSubnets() { if nodeRef := os.Getenv("CALICO_K8S_NODE_REF"); nodeRef != "" { k8sNodeName = nodeRef } - if config, err = clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")); err == nil { + kubeconfig := os.Getenv("KUBECONFIG") + // Host env vars may override the container on Windows HPC, so $env:KUBECONFIG cannot + // be trusted in this case + // FIXME: this will no longer be needed when containerd v1.6 is EOL'd + if winutils.InHostProcessContainer() { + kubeconfig = "" + } + if config, err = winutils.BuildConfigFromFlags("", kubeconfig); err == nil { // Create the k8s clientset. clientset, err = kubernetes.NewForConfig(config) if err != nil { diff --git a/node/pkg/winupgrade/k8s_resources.go b/node/pkg/winupgrade/k8s_resources.go deleted file mode 100644 index f7da6050c83..00000000000 --- a/node/pkg/winupgrade/k8s_resources.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright (c) 2021 Tigera, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package winupgrade - -import ( - "context" - "time" - - apierrs "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" -) - -// k8snode holds a collection of helper functions for Kubernetes node. -type k8snode string - -// Add / remove node annotations to node. Perform Get/Check/Update so that it always working on latest version. -// If node labels has been set already, do nothing. -func (n k8snode) addRemoveNodeAnnotations(k8sClientset *kubernetes.Clientset, - toAdd map[string]string, - toRemove []string) error { - nodeName := string(n) - return wait.PollImmediate(3*time.Second, 1*time.Minute, func() (bool, error) { - node, err := k8sClientset.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{}) - if err != nil { - return false, err - } - - needUpdate := false - for k, v := range toAdd { - if currentVal, ok := node.Annotations[k]; ok && currentVal == v { - continue - } - node.Annotations[k] = v - needUpdate = true - } - - for _, k := range toRemove { - if _, ok := node.Annotations[k]; ok { - delete(node.Annotations, k) - needUpdate = true - } - } - - if needUpdate { - _, err := k8sClientset.CoreV1().Nodes().UpdateStatus(context.Background(), node, metav1.UpdateOptions{}) - if err == nil { - return true, nil - } - if !apierrs.IsConflict(err) { - return false, err - } - - // Retry on update conflicts. - return false, nil - } - - // no update needed - return true, nil - }) -} diff --git a/node/pkg/winupgrade/should_upgrade.go b/node/pkg/winupgrade/should_upgrade.go deleted file mode 100644 index b2904cc7a07..00000000000 --- a/node/pkg/winupgrade/should_upgrade.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (c) 2021 Tigera, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package winupgrade - -import ( - "context" - "os" - "strings" - - "github.com/projectcalico/calico/libcalico-go/lib/names" - - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" - - "github.com/projectcalico/calico/node/pkg/lifecycle/utils" - - log "github.com/sirupsen/logrus" -) - -// Exit with code zero if Windows upgrade service should be installed. -func ShouldInstallUpgradeService() { - version := getVersion() - variant := getVariant() - - // Determine the name for this node. - nodeName := determineNodeName() - log.Debugf("Check if Calico upgrade service should be installed on node: %s. Version: %s, Variant: %s, baseDir: %s", nodeName, version, variant, baseDir()) - - config, err := clientcmd.BuildConfigFromFlags("", kubeConfigFile()) - if err != nil { - log.WithError(err).Fatal("Failed to build Kubernetes client config") - os.Exit(2) - } - clientSet, err := kubernetes.NewForConfig(config) - if err != nil { - log.WithError(err).Fatal("Failed to create Kubernetes client") - os.Exit(2) - } - - // Update annotations for running variant and version. - node := k8snode(nodeName) - err = node.addRemoveNodeAnnotations(clientSet, - map[string]string{ - CalicoVersionAnnotation: version, - CalicoVariantAnnotation: variant, - }, - []string{}) - if err != nil { - log.WithError(err).Fatal("Failed to set version/variant annotations") - os.Exit(2) - } - - upgrade, _ := upgradeTriggered(context.Background(), clientSet, nodeName) - if !upgrade { - os.Exit(1) - } - os.Exit(0) -} - -// DetermineNodeName is copied from utils package but with logging in DEBUG level. -func determineNodeName() string { - var nodeName string - var err error - - // Determine the name of this node. Precedence is: - // - NODENAME - // - Value stored in our nodename file. - // - HOSTNAME (lowercase) - // - os.Hostname (lowercase). - // We use the names.Hostname which lowercases and trims the name. - if nodeName = strings.TrimSpace(os.Getenv("NODENAME")); nodeName != "" { - log.Debugf("Using NODENAME environment for node name %s", nodeName) - } else if nodeName = utils.NodenameFromFile(); nodeName != "" { - log.Debugf("Using stored node name %s", nodeName) - } else if nodeName = strings.ToLower(strings.TrimSpace(os.Getenv("HOSTNAME"))); nodeName != "" { - log.Debugf("Using HOSTNAME environment (lowercase) for node name %s", nodeName) - } else if nodeName, err = names.Hostname(); err != nil { - log.WithError(err).Error("Unable to determine hostname") - utils.Terminate() - } else { - log.Warn("Using auto-detected node name. It is recommended that an explicit value is supplied using " + - "the NODENAME environment variable.") - } - log.Debugf("Determined node name: %s", nodeName) - - return nodeName -} diff --git a/node/pkg/winupgrade/upgrade.go b/node/pkg/winupgrade/upgrade.go deleted file mode 100644 index 27d68ebdb7e..00000000000 --- a/node/pkg/winupgrade/upgrade.go +++ /dev/null @@ -1,411 +0,0 @@ -// Copyright (c) 2021 Tigera, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package winupgrade - -import ( - "bytes" - "context" - "fmt" - "os" - "os/exec" - "os/signal" - "path/filepath" - "strings" - "syscall" - "time" - - image "github.com/distribution/distribution/reference" - - "github.com/projectcalico/calico/node/pkg/lifecycle/startup" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" - - "github.com/projectcalico/calico/node/pkg/lifecycle/utils" - - log "github.com/sirupsen/logrus" -) - -const ( - CalicoKubeConfigFile = "calico-kube-config" - CalicoUpgradeDir = "c:\\CalicoUpgrade" - EnterpriseDir = "TigeraCalico" - CalicoUpgradeLabel = "projectcalico.org/windows-upgrade" - CalicoUpgradeInProgress = "in-progress" - CalicoVersionAnnotation = "projectcalico.org/version" - CalicoVariantAnnotation = "projectcalico.org/variant" - CalicoUpgradeScript = "calico-upgrade.ps1" -) - -func getVariant() string { - if runningEnterprise() { - return "TigeraSecureEnterprise" - } - return "Calico" -} - -func getVersion() string { - return startup.VERSION -} - -// This file contains the upgrade processing for the calico/node. This -// includes: -// - Monitoring node labels and getting the Calico Windows upgrade script file from the label. -// - Uninstalling current Calico Windows (OSS or Enterprise) running on the node. -// - Install new version of Calico Windows. -func Run() { - version := getVersion() - variant := getVariant() - - // Determine the name for this node. - nodeName := utils.DetermineNodeName() - log.Infof("Starting Calico upgrade service on node: %s. Version: %s, Variant: %s, baseDir: %s", nodeName, version, variant, baseDir()) - - config, err := clientcmd.BuildConfigFromFlags("", kubeConfigFile()) - if err != nil { - log.WithError(err).Fatal("Failed to build Kubernetes client config") - } - clientSet, err := kubernetes.NewForConfig(config) - if err != nil { - log.WithError(err).Fatal("Failed to create Kubernetes client") - } - - stdout, stderr, err := powershell("Get-ComputerInfo | select WindowsVersion, OsBuildNumber, OsHardwareAbstractionLayer") - fmt.Println(stdout, stderr) - if err != nil { - log.WithError(err).Fatal("Failed to interact with powershell") - } - - ctx, cancel := context.WithCancel(context.Background()) - go loop(ctx, clientSet, nodeName) - - // Trap cancellation on Windows. https://golang.org/pkg/os/signal/ - sigCh := make(chan os.Signal, 1) - signal.Notify( - sigCh, - syscall.SIGTERM, - syscall.SIGINT, - syscall.SIGQUIT, - ) - - <-sigCh - cancel() - log.Info("Received system signal to exit") -} - -func loop(ctx context.Context, cs kubernetes.Interface, nodeName string) { - ticker := time.NewTicker(10 * time.Second) - upgradeScript := filepath.Join(CalicoUpgradeDir, CalicoUpgradeScript) - - for { - select { - case <-ctx.Done(): - ticker.Stop() - return - case <-ticker.C: - upgrade, err := upgradeTriggered(ctx, cs, nodeName) - if err != nil { - log.WithError(err).Error("Failed to check node upgrade status, will retry") - break - } - // If upgrade not triggered yet just silently continue. - if !upgrade { - break - } - - if !pathExists(upgradeScript) { - log.Info("Upgrade triggered but upgrade artifacts not ready yet, will retry") - break - } - - log.Info("Calico upgrade process is starting") - - // Before executing the script, verify host path volume mount. - err = verifyPodImageWithHostPathVolume(cs, nodeName, CalicoUpgradeDir) - if err != nil { - log.WithError(err).Fatal("Failed to verify windows-upgrade pod image") - } - - err = uninstall() - if err != nil { - log.WithError(err).Error("Uninstall failed, will retry") - break - } - - time.Sleep(3 * time.Second) - err = execScript(upgradeScript) - if err != nil { - log.WithError(err).Fatal("Failed to upgrade to new version") - } - - // Upgrade will run in another process. The running - // calico-upgrade service is done. The new calico-upgrade - // service will clean the old service up. - date := time.Now().Format("2006-01-02") - log.Info(fmt.Sprintf("Upgrade is in progress. Upgrade log is in c:\\calico-upgrade.%v.log", date)) - time.Sleep(3 * time.Second) - return - } - } -} - -func upgradeTriggered(ctx context.Context, cs kubernetes.Interface, nodeName string) (bool, error) { - node, err := cs.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) - // Return error getting node info. - if err != nil { - return false, fmt.Errorf("Could not get node resource: %w", err) - } - - upgradeStatus, ok := node.Labels[CalicoUpgradeLabel] - // Upgrade label doesn't exist yet. - if !ok { - return false, nil - } - if upgradeStatus != CalicoUpgradeInProgress { - return false, fmt.Errorf("Unexpected upgrade status label value: %v", upgradeStatus) - } - - return true, nil -} - -func pathExists(path string) bool { - if _, err := os.Stat(path); err != nil { - return false - } - - return true -} - -// Return the base directory for Calico upgrade service. -func baseDir() string { - dir := filepath.Dir(os.Args[0]) - return "c:\\" + filepath.Base(dir) -} - -// Return if the monitor service is running as part of Enterprise installation. -func runningEnterprise() bool { - return strings.Contains(baseDir(), EnterpriseDir) -} - -// Return kubeconfig file path for Calico -func kubeConfigFile() string { - return baseDir() + "\\" + CalicoKubeConfigFile -} - -func uninstall() error { - path := filepath.Join(baseDir(), "uninstall-calico.ps1") - log.Infof("Start uninstall script %s\n", path) - stdout, stderr, err := powershell(path + " -ExceptUpgradeService $true") - fmt.Println(stdout, stderr) - if err != nil { - return err - } - // After the uninstall completes, move the existing calico-node.exe to - // a temporary file. The calico-upgrade service is still running so not - // doing this means we cannot replace calico-node.exe with the upgrade. - stdout, stderr, err = powershell(fmt.Sprintf(`mv %v\calico-node.exe %v\calico-node.exe.to-be-replaced`, baseDir(), baseDir())) - fmt.Println(stdout, stderr) - if err != nil { - return err - } - - return nil -} - -func execScript(script string) error { - log.Infof("Start script %s\n", script) - - // This has to be done in a separate process because when the new calico services are started, the existing - // calico-upgrade service is removed so the new calico-upgrade service can be started. - // However, removing the existing calico-upgrade service means the powershell - // process running the upgrade script is killed and the installation is left - // incomplete. - cmd := fmt.Sprintf(`Start-Process powershell -argument %q -WindowStyle hidden`, script) - stdout, stderr, err := powershell(cmd) - - if err != nil { - return err - } - fmt.Println(stdout, stderr) - return nil -} - -func verifyImagesSharePathPrefix(first, second string) error { - n1, err := image.ParseNamed(first) - if err != nil { - return err - } - n2, err := image.ParseNamed(second) - if err != nil { - return err - } - // Compare the domain parts (e.g. docker.io) of the image references. - // Domain is always present, ParseNamed fails without a domain. - if image.Domain(n1) != image.Domain(n2) { - return fmt.Errorf("images %q and %q do not share the same domain", first, second) - } - - // Split the image path. E.g. if the image is - // "docker.io/calico/node:v3.21.0" then the path is "calico/node". - n1PathParts := strings.Split(image.Path(n1), "/") - n2PathParts := strings.Split(image.Path(n2), "/") - - // Special case: if the image references are both short image references - // like my-registry.org/node:v3.21.0 then the path will just be the image - // component itself. In this case, we can only compare the domain. - if len(n1PathParts) == 1 && len(n2PathParts) == 1 { - return nil - } - - // Validation should catch this but just in case. - if len(n1PathParts) == 0 || len(n2PathParts) == 0 { - return fmt.Errorf("images %q and/or %q do not contain a valid path", first, second) - } - - // If image paths do not have equal # of segments then they don't match. - if len(n1PathParts) != len(n2PathParts) { - return fmt.Errorf("images %q and %q do not share the same path prefix", first, second) - } - - // Remove the last segment of the image path since it will contain the - // component name. - n1PathPrefix := n1PathParts[:len(n1PathParts)-1] - n2PathPrefix := n2PathParts[:len(n2PathParts)-1] - - // Compare the image path prefix (e.g. "docker.io/calico", "quay.io/calico") - // of the two images - for i := range n1PathPrefix { - if n1PathPrefix[i] != n2PathPrefix[i] { - return fmt.Errorf("images %q and %q are not from the same registry and path", first, second) - } - } - return nil -} - -func verifyPodImageWithHostPathVolume(cs kubernetes.Interface, nodeName string, hostPath string) error { - // Get pod list for calico-system pods. - list, err := cs.CoreV1().Pods("calico-system").List(context.TODO(), metav1.ListOptions{}) - if err != nil { - return err - } - - // Find the calico-node pod. - var calicoPod v1.Pod - var found bool - for _, pod := range list.Items { - if strings.HasPrefix(pod.Name, "calico-node") && pod.Spec.ServiceAccountName == "calico-node" { - calicoPod = pod - found = true - break - } - } - - if !found { - return fmt.Errorf("could not find a calico-node pod") - } - - // Ensure the pod is owned by the calico-node daemonset. - var ownedByCalicoDs bool - for _, ownerRef := range calicoPod.ObjectMeta.OwnerReferences { - if ownerRef.Kind == "DaemonSet" && ownerRef.Name == "calico-node" && *ownerRef.Controller { - ownedByCalicoDs = true - } - } - if !ownedByCalicoDs { - return fmt.Errorf("calico-node pod not owned by calico-node daemonset") - } - - // Get the calico node nodeImage - var nodeImage string - for _, c := range calicoPod.Spec.Containers { - if c.Name == "calico-node" { - nodeImage = c.Image - break - } - } - if nodeImage == "" { - return fmt.Errorf("calico-node container image not found") - } - - log.Infof("Found node container image is %v", nodeImage) - - hasHostPathVolume := func(pod v1.Pod, hostPath string) bool { - for _, v := range pod.Spec.Volumes { - path := v.HostPath - if path == nil { - continue - } - if path.Path == hostPath { - return true - } - } - return false - } - - var podWithHostPath v1.Pod - count := 0 - for _, pod := range list.Items { - // Only look at pods on this node. - if pod.Spec.NodeName != nodeName { - continue - } - if hasHostPathVolume(pod, hostPath) { - podWithHostPath = pod - count++ - } - } - - if count == 0 { - return fmt.Errorf("Failed to find pod with expected host path") - } - - if count > 1 { - return fmt.Errorf("More than one pod has expected host path") - } - - if len(podWithHostPath.Spec.Containers) != 1 { - return fmt.Errorf("Pod with hostpath volume has more than one container") - } - upgradeImage := podWithHostPath.Spec.Containers[0].Image - log.Infof("Found upgrade image: %v", upgradeImage) - err = verifyImagesSharePathPrefix(nodeImage, upgradeImage) - if err != nil { - return err - } - return nil -} - -func powershell(args ...string) (string, string, error) { - ps, err := exec.LookPath("powershell.exe") - if err != nil { - return "", "", err - } - - args = append([]string{"-NoProfile", "-NonInteractive"}, args...) - cmd := exec.Command(ps, args...) - - var stdout bytes.Buffer - var stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - - err = cmd.Run() - if err != nil { - return "", "", err - } - - return stdout.String(), stderr.String(), err -} diff --git a/node/pkg/winupgrade/upgrade_suite_test.go b/node/pkg/winupgrade/upgrade_suite_test.go deleted file mode 100644 index 6ba2379a458..00000000000 --- a/node/pkg/winupgrade/upgrade_suite_test.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (c) 2021 Tigera, Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package winupgrade_test - -import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - "testing" - - "github.com/onsi/ginkgo/reporters" - - "github.com/projectcalico/calico/libcalico-go/lib/testutils" -) - -func init() { - testutils.HookLogrusForGinkgo() -} - -func TestCommands(t *testing.T) { - RegisterFailHandler(Fail) - junitReporter := reporters.NewJUnitReporter("../../report/upgrade_suite.xml") - RunSpecsWithDefaultAndCustomReporters(t, "Upgrade Suite", []Reporter{junitReporter}) -} diff --git a/node/pkg/winupgrade/upgrade_test.go b/node/pkg/winupgrade/upgrade_test.go deleted file mode 100644 index d9a59837237..00000000000 --- a/node/pkg/winupgrade/upgrade_test.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (c) 2021-2022 Tigera, Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package winupgrade - -import ( - . "github.com/onsi/ginkgo/extensions/table" - . "github.com/onsi/gomega" -) - -const ( - nodeShortImageTag = "example.com/node:v3.21.0" - nodeDockerImageTag = "docker.io/calico/node:v3.21.0" - nodeDockerImageDigest = "docker.io/calico/node@sha256:1a54e9ad69451473fde398ac63a5f5696712cf38ed00f0deadc4189927b93176" - nodeQuayImageDigest = "quay.io/calico/node@sha256:bf87045cbb6c3f9ca39b9724350f728e8dab780b3e6185d413c7f232fb0452bf" - - windowsShortImageTag = "example.com/calico-windows-upgrade:v3.21.0" - windowsShortImageTag2 = "my-registry.org/calico-windows-upgrade:v3.21.0" - windowsDockerImageTag = "docker.io/calico/windows-upgrade:v3.21.0" - windowsDockerImageDigest = "docker.io/calico/windows-upgrade@sha256:1aa17a74e3f084e94b0d1f93bdd745c8c88cbb292907ac4fa94d6f206a5e49db" - windowsQuayImageTag = "quay.io/calico/windows-upgrade:v3.21.0" - - nodeLongImageTag = "example.com/tigera/calico/testing/node:v3.21.0" - windowsLongImageTag = "example.com/tigera/calico/windows-upgrade:v3.21.0" -) - -var _ = DescribeTable("verifyImagesShareRegistryPath", - func(upgradeImage string, nodeImage string, noError bool) { - err := verifyImagesSharePathPrefix(upgradeImage, nodeImage) - Expect(err == nil).To(Equal(noError)) - }, - Entry("same prefix, tag", windowsDockerImageTag, nodeDockerImageTag, true), - Entry("same prefix, digest1", windowsDockerImageTag, nodeDockerImageDigest, true), - Entry("same prefix, digest2", windowsDockerImageDigest, nodeDockerImageDigest, true), - Entry("same prefix, short tags", nodeShortImageTag, windowsShortImageTag, true), - - Entry("diff prefix, tag", windowsQuayImageTag, nodeDockerImageTag, false), - Entry("diff prefix, digest1", windowsDockerImageTag, nodeQuayImageDigest, false), - Entry("diff prefix, digest2", nodeDockerImageDigest, nodeQuayImageDigest, false), - Entry("diff prefix, short tags", nodeShortImageTag, windowsShortImageTag2, false), - Entry("diff prefix, diff lengths", nodeLongImageTag, windowsLongImageTag, false), -) diff --git a/node/windows-packaging/CalicoWindows/confd/confd-service.ps1 b/node/windows-packaging/CalicoWindows/confd/confd-service.ps1 index 0994b17e720..2988c8358e6 100644 --- a/node/windows-packaging/CalicoWindows/confd/confd-service.ps1 +++ b/node/windows-packaging/CalicoWindows/confd/confd-service.ps1 @@ -24,7 +24,7 @@ if ($env:CNI_IPAM_TYPE -EQ "host-local") { $env:USE_POD_CIDR = "false" } -if($env:CALICO_NETWORKING_BACKEND = "windows-bgp") +if($env:CALICO_NETWORKING_BACKEND -EQ "windows-bgp") { Wait-ForCalicoInit Write-Host "Windows BGP is enabled, running confd..." @@ -45,4 +45,3 @@ if($env:CALICO_NETWORKING_BACKEND = "windows-bgp") Start-Sleep 10 } } - diff --git a/node/windows-packaging/CalicoWindows/config-hpc.ps1 b/node/windows-packaging/CalicoWindows/config-hpc.ps1 new file mode 100644 index 00000000000..67519dde9f8 --- /dev/null +++ b/node/windows-packaging/CalicoWindows/config-hpc.ps1 @@ -0,0 +1,75 @@ +# Copyright (c) 2023 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +$baseDir = "$PSScriptRoot" +ipmo $baseDir\libs\calico\calico.psm1 -Force + +Write-Host "Setting environment variables if not set..." + +## Cluster configuration: + +# KUBE_NETWORK should be set to a regular expression that matches the HNS network(s) used for pods. +# The default, "Calico.*", is correct for Calico CNI. +Set-EnvVarIfNotSet -var "KUBE_NETWORK" -defaultValue "Calico.*" + +# Set this to one of the following values: +# - "vxlan" for Calico VXLAN networking +# - "windows-bgp" for Calico BGP networking using the Windows BGP router. +# - "none" to disable the Calico CNI plugin (so that you can use another plugin). +Set-EnvVarIfNotSet -var "CALICO_NETWORKING_BACKEND" -defaultValue "vxlan" + +# Set to match your Kubernetes service CIDR. +Set-EnvVarIfNotSet -var "DNS_SEARCH" -defaultValue "svc.cluster.local" + +## VXLAN-specific configuration. + +# The VXLAN VNI / VSID. Must match the VXLANVNI felix configuration parameter used +# for Linux nodes. +Set-EnvVarIfNotSet -var "VXLAN_VNI" -defaultValue 4096 +# Prefix used when generating MAC addresses for virtual NICs. +Set-EnvVarIfNotSet -var "VXLAN_MAC_PREFIX" -defaultValue "0E-2A" +# Network Adapter used on VXLAN, leave blank for primary NIC. +Set-EnvVarIfNotSet -var "VXLAN_ADAPTER" -defaultValue "" + +## Node configuration. + +# The NODENAME variable should be set to match the Kubernetes Node name of this host. +# The default uses this node's hostname (which is the same as kubelet). +# +# Note: on AWS, kubelet is often configured to use the internal domain name of the host rather than +# the simple hostname, for example "ip-172-16-101-135.us-west-2.compute.internal". +Set-EnvVarIfNotSet -var "NODENAME" -defaultValue $(hostname).ToLower() +# Similarly, CALICO_K8S_NODE_REF should be set to the Kubernetes Node name. When using etcd, +# the Calico kube-controllers pod will clean up Calico node objects if the corresponding Kubernetes Node is +# cleaned up. +Set-EnvVarIfNotSet -var "CALICO_K8S_NODE_REF" -defaultValue $env:NODENAME + +# The time out to wait for a valid IP of an interface to be assigned before initialising Calico +# after a reboot. +Set-EnvVarIfNotSet -var "STARTUP_VALID_IP_TIMEOUT" -defaultValue 90 + +# The IP of the node; the default will auto-detect a usable IP in most cases. +Set-EnvVarIfNotSet -var "IP" -defaultValue "autodetect" + +# Felix logs to screen at info level by default. Uncomment this line to override the log +# level. Alternatively, (if this is commented out) the log level can be controlled via +# the FelixConfiguration resource in the datastore. +# $env:FELIX_LOGSEVERITYSCREEN = "info" +# Disable logging to file by default since the service wrapper will redirect our log to file. +Set-EnvVarIfNotSet -var "FELIX_LOGSEVERITYFILE" -defaultValue "none" +# Disable syslog logging, which is not supported on Windows. +Set-EnvVarIfNotSet -var "FELIX_LOGSEVERITYSYS" -defaultValue "none" +# confd logs to screen at info level by default. Uncomment this line to override the log +# level. +#Set-EnvVarIfNotSet -var "BGP_LOGSEVERITYSCREEN" -defaultValue "debug" diff --git a/node/windows-packaging/CalicoWindows/config.ps1 b/node/windows-packaging/CalicoWindows/config.ps1 index 4597c0360e5..80203067379 100644 --- a/node/windows-packaging/CalicoWindows/config.ps1 +++ b/node/windows-packaging/CalicoWindows/config.ps1 @@ -1,3 +1,17 @@ +# Copyright (c) 2021-2023 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + $baseDir = "$PSScriptRoot" ipmo $baseDir\libs\calico\calico.psm1 -Force diff --git a/node/windows-packaging/CalicoWindows/uninstall-calico-hpc.ps1 b/node/windows-packaging/CalicoWindows/uninstall-calico-hpc.ps1 new file mode 100644 index 00000000000..d9d596438c0 --- /dev/null +++ b/node/windows-packaging/CalicoWindows/uninstall-calico-hpc.ps1 @@ -0,0 +1,95 @@ +# Copyright (c) 2023 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +<# +.DESCRIPTION + This script stops Calico services and fully uninstalls Calico on a Windows node. +#> + +Param( + [parameter(Mandatory = $false)] $NSSMPath="$env:CONTAINER_SANDBOX_MOUNT_POINT/nssm.exe" +) + +$ErrorActionPreference = 'SilentlyContinue' + +function Remove-CalicoService($ServiceName) +{ + $svc = Get-Service | where Name -EQ "$ServiceName" + if ($svc -NE $null) + { + if ($svc.Status -EQ 'Running') + { + Write-Host "$ServiceName service is running, stopping it..." + & $NSSMPath stop $ServiceName confirm + } + Write-Host "Removing $ServiceName service..." + & $NSSMPath remove $ServiceName confirm + } +} + +if ("$env:CNI_NET_DIR" -eq $null) +{ + Write-Host "CNI_NET_DIR env var not set, skipping Calico CNI config cleanup" +} elseif (-not (Test-Path "$env:CONTAINER_SANDBOX_MOUNT_POINT/$env:CNI_NET_DIR")) +{ + Write-Host "$env:CNI_NET_DIR dir does not exist, skipping Calico CNI config cleanup" +} else +{ + $cniConfFile = "$env:CONTAINER_SANDBOX_MOUNT_POINT/$env:CNI_NET_DIR/10-calico.conf" + if (Test-Path $cniConfFile) { + Write-Host "Removing Calico CNI conf file at $cniConfFile ..." + rm $cniConfFile + } + + if (Test-Path "$env:CNI_CONF_NAME") { + $cniConfListFile = "$env:CONTAINER_SANDBOX_MOUNT_POINT/$env:CNI_NET_DIR/$env:CNI_CONF_NAME" + if (Test-Path $cniConfListFile) { + Write-Host "Removing Calico CNI conf file at $cniConfListFile ..." + rm $cniConfListFile + } + } +} + + +if ("$env:CNI_BIN_DIR" -eq $null) +{ + Write-Host "CNI_BIN_DIR env var not set, skipping Calico CNI binary cleanup" +} elseif (-not (Test-Path "$env:CONTAINER_SANDBOX_MOUNT_POINT/$env:CNI_BIN_DIR")) +{ + Write-Host "$env:CNI_BIN_DIR dir does not exist, skipping Calico CNI binary cleanup" +} else +{ + $cniBinPath = "$env:CONTAINER_SANDBOX_MOUNT_POINT/$env:CNI_BIN_DIR/calico*.exe" + if (Test-Path $cniBinPath) { + Write-Host "Removing Calico CNI binaries at $cniBinPath ..." + rm $cniBinPath + } +} + +Write-Host "Stopping and removing Calico services if they are present..." +Remove-CalicoService CalicoConfd +Remove-CalicoService CalicoFelix +Remove-CalicoService CalicoNode +Remove-CalicoService CalicoUpgrade + +Write-Host "Stopping and removing kube-proxy service if it is present..." +Write-Host "It is recommended to run kube-proxy as kubernetes daemonset instead" +Remove-CalicoService kube-proxy + +Write-Host "Logging containerd CNI bin and conf dir paths:" +Get-Content "$env:ProgramFiles/containerd/config.toml" | Select-String -Pattern "^(\s)*bin_dir = (.)*$" +Get-Content "$env:ProgramFiles/containerd/config.toml" | Select-String -Pattern "^(\s)*conf_dir = (.)*$" + +Get-Module 'calico' | Remove-Module -Force +Write-Host "Done." diff --git a/typha/pkg/discovery/discovery.go b/typha/pkg/discovery/discovery.go index 654b6a04ca1..4a503a78cfb 100644 --- a/typha/pkg/discovery/discovery.go +++ b/typha/pkg/discovery/discovery.go @@ -24,9 +24,9 @@ import ( "github.com/sirupsen/logrus" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" "github.com/projectcalico/calico/libcalico-go/lib/set" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" "github.com/projectcalico/calico/libcalico-go/lib/seedrng" ) @@ -177,7 +177,7 @@ func (d *Discoverer) discoverTyphaAddrs() ([]Typha, error) { if d.k8sClient == nil && d.inCluster { // Client didn't provide a kube client but we're allowed to create one. logrus.Info("Creating Kubernetes client for Typha discovery...") - k8sConf, err := rest.InClusterConfig() + k8sConf, err := winutils.GetInClusterConfig() if err != nil { logrus.WithError(err).Error("Unable to create in-cluster Kubernetes config.") return nil, err diff --git a/typha/pkg/k8s/lookup.go b/typha/pkg/k8s/lookup.go index 340c86d23ab..c6242e2fb3a 100644 --- a/typha/pkg/k8s/lookup.go +++ b/typha/pkg/k8s/lookup.go @@ -20,9 +20,9 @@ import ( log "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" "github.com/projectcalico/calico/libcalico-go/lib/set" + "github.com/projectcalico/calico/libcalico-go/lib/winutils" "github.com/projectcalico/calico/typha/pkg/calc" ) @@ -38,7 +38,7 @@ type RealK8sAPI struct { func (r *RealK8sAPI) clientSet() (*kubernetes.Clientset, error) { if r.cachedClientSet == nil { // TODO Typha: support Typha lookup without using rest.InClusterConfig(). - k8sconf, err := rest.InClusterConfig() + k8sconf, err := winutils.GetInClusterConfig() if err != nil { log.WithError(err).Error("Unable to create Kubernetes config.") return nil, err