Merge pull request kubernetes#104015 from neolit123/1.23-use-dynamic-versions

kubeadm: dynamically populate the current/minimum k8s versions

Author: k8s-ci-robot

cmd/kubeadm/app/cmd/config_test.go

@@ -208,8 +208,8 @@ func TestConfigImagesListRunWithoutPath(t *testing.T) {
 
 func TestConfigImagesListOutput(t *testing.T) {
 
-	etcdVersion, ok := constants.SupportedEtcdVersion[uint8(dummyKubernetesVersion.Minor())]
-	if !ok {
+	etcdVersion, _, err := constants.EtcdSupportedVersion(constants.SupportedEtcdVersion, dummyKubernetesVersionStr)
+	if err != nil {
 		t.Fatalf("cannot determine etcd version for Kubernetes version %s", dummyKubernetesVersionStr)
 	}
 	versionMapping := struct {
@@ -218,7 +218,7 @@ func TestConfigImagesListOutput(t *testing.T) {
 		PauseVersion   string
 		CoreDNSVersion string
 	}{
-		EtcdVersion:    etcdVersion,
+		EtcdVersion:    etcdVersion.String(),
 		KubeVersion:    "v" + dummyKubernetesVersionStr,
 		PauseVersion:   constants.PauseVersion,
 		CoreDNSVersion: constants.CoreDNSVersion,

cmd/kubeadm/app/constants/constants.go

@@ -29,7 +29,9 @@ import (
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/apimachinery/pkg/util/wait"
+	apimachineryversion "k8s.io/apimachinery/pkg/version"
 	bootstrapapi "k8s.io/cluster-bootstrap/token/api"
+	componentversion "k8s.io/component-base/version"
 	utilnet "k8s.io/utils/net"
 
 	"github.com/pkg/errors"
@@ -448,13 +450,13 @@ var (
 	ControlPlaneComponents = []string{KubeAPIServer, KubeControllerManager, KubeScheduler}
 
 	// MinimumControlPlaneVersion specifies the minimum control plane version kubeadm can deploy
-	MinimumControlPlaneVersion = version.MustParseSemantic("v1.21.0")
+	MinimumControlPlaneVersion = getSkewedKubernetesVersion(-1)
 
 	// MinimumKubeletVersion specifies the minimum version of kubelet which kubeadm supports
-	MinimumKubeletVersion = version.MustParseSemantic("v1.21.0")
+	MinimumKubeletVersion = getSkewedKubernetesVersion(-1)
 
 	// CurrentKubernetesVersion specifies current Kubernetes version supported by kubeadm
-	CurrentKubernetesVersion = version.MustParseSemantic("v1.22.0")
+	CurrentKubernetesVersion = getSkewedKubernetesVersion(0)
 
 	// SupportedEtcdVersion lists officially supported etcd versions with corresponding Kubernetes releases
 	SupportedEtcdVersion = map[uint8]string{
@@ -483,8 +485,52 @@ var (
 		Factor:   1.0,
 		Jitter:   0.1,
 	}
+
+	// defaultKubernetesVersionForTests is the default version used for unit tests.
+	// The MINOR should be at least 3 as some tests subtract 3 from the MINOR version.
+	defaultKubernetesVersionForTests = version.MustParseSemantic("v1.3.0")
 )
 
+// isRunningInTest can be used to determine if the code in this file is being run in a test.
+func isRunningInTest() bool {
+	return strings.HasSuffix(os.Args[0], ".test")
+}
+
+// getSkewedKubernetesVersion returns the current MAJOR.(MINOR+n).0 Kubernetes version with a skew of 'n'
+// It uses the kubeadm version provided by the 'component-base/version' package. This version must be populated
+// by passing linker flags during the kubeadm build process. If the version is empty, assume that kubeadm
+// was either build incorrectly or this code is running in unit tests.
+func getSkewedKubernetesVersion(n int) *version.Version {
+	versionInfo := componentversion.Get()
+	ver := getSkewedKubernetesVersionImpl(&versionInfo, n, isRunningInTest)
+	if ver == nil {
+		panic("kubeadm is not build properly using 'make ...': missing component version information")
+	}
+	return ver
+}
+
+func getSkewedKubernetesVersionImpl(versionInfo *apimachineryversion.Info, n int, isRunningInTestFunc func() bool) *version.Version {
+	// TODO: update if the kubeadm version gets decoupled from the Kubernetes version.
+	// This would require keeping track of the supported skew in a table.
+	// More changes would be required if the kubelet version one day decouples from that of Kubernetes.
+	var ver *version.Version
+	if len(versionInfo.Major) == 0 {
+		if isRunningInTestFunc() {
+			ver = defaultKubernetesVersionForTests // An arbitrary version for testing purposes
+		} else {
+			// If this is not running in tests assume that the kubeadm binary is not build properly
+			return nil
+		}
+	} else {
+		ver = version.MustParseSemantic(versionInfo.GitVersion)
+	}
+	// Append the MINOR version skew.
+	// TODO: handle the case of Kubernetes moving to v2.0 or having MAJOR version updates in the future.
+	// This would require keeping track (in a table) of the last MINOR for a particular MAJOR.
+	minor := uint(int(ver.Minor()) + n)
+	return version.MustParseSemantic(fmt.Sprintf("v%d.%d.0", ver.Major(), minor))
+}
+
 // EtcdSupportedVersion returns officially supported version of etcd for a specific Kubernetes release
 // If passed version is not in the given list, the function returns the nearest version with a warning
 func EtcdSupportedVersion(supportedEtcdVersion map[uint8]string, versionString string) (etcdVersion *version.Version, warning, err error) {

cmd/kubeadm/app/constants/constants_test.go

@@ -21,6 +21,7 @@ import (
 	"testing"
 
 	"k8s.io/apimachinery/pkg/util/version"
+	apimachineryversion "k8s.io/apimachinery/pkg/version"
 )
 
 func TestGetStaticPodDirectory(t *testing.T) {
@@ -237,3 +238,61 @@ func TestGetKubernetesServiceCIDR(t *testing.T) {
 		})
 	}
 }
+
+func TestGetSkewedKubernetesVersionImpl(t *testing.T) {
+	tests := []struct {
+		name                string
+		versionInfo         *apimachineryversion.Info
+		n                   int
+		isRunningInTestFunc func() bool
+		expectedResult      *version.Version
+	}{
+		{
+			name:           "invalid versionInfo; running in test",
+			versionInfo:    &apimachineryversion.Info{},
+			expectedResult: defaultKubernetesVersionForTests,
+		},
+		{
+			name:                "invalid versionInfo; not running in test",
+			versionInfo:         &apimachineryversion.Info{},
+			isRunningInTestFunc: func() bool { return false },
+			expectedResult:      nil,
+		},
+		{
+			name:           "valid skew of -1",
+			versionInfo:    &apimachineryversion.Info{Major: "1", GitVersion: "v1.23.0"},
+			n:              -1,
+			expectedResult: version.MustParseSemantic("v1.22.0"),
+		},
+		{
+			name:           "valid skew of 0",
+			versionInfo:    &apimachineryversion.Info{Major: "1", GitVersion: "v1.23.0"},
+			n:              0,
+			expectedResult: version.MustParseSemantic("v1.23.0"),
+		},
+		{
+			name:           "valid skew of +1",
+			versionInfo:    &apimachineryversion.Info{Major: "1", GitVersion: "v1.23.0"},
+			n:              1,
+			expectedResult: version.MustParseSemantic("v1.24.0"),
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			if tc.isRunningInTestFunc == nil {
+				tc.isRunningInTestFunc = func() bool { return true }
+			}
+			result := getSkewedKubernetesVersionImpl(tc.versionInfo, tc.n, tc.isRunningInTestFunc)
+			if (tc.expectedResult == nil) != (result == nil) {
+				t.Errorf("expected result: %v, got: %v", tc.expectedResult, result)
+			}
+			if result == nil {
+				return
+			}
+			if cmp, _ := result.Compare(tc.expectedResult.String()); cmp != 0 {
+				t.Errorf("expected result: %v, got %v", tc.expectedResult, result)
+			}
+		})
+	}
+}

cmd/kubeadm/app/phases/upgrade/compute_test.go

@@ -85,7 +85,8 @@ spec:
     image: k8s.gcr.io/etcd:` + fakeCurrentEtcdVersion
 
 func getEtcdVersion(v *versionutil.Version) string {
-	return constants.SupportedEtcdVersion[uint8(v.Minor())]
+	etcdVer, _, _ := constants.EtcdSupportedVersion(constants.SupportedEtcdVersion, v.String())
+	return etcdVer.String()
 }
 
 const fakeCurrentCoreDNSVersion = "1.0.6"

cmd/kubeadm/app/phases/upgrade/policy_test.go

@@ -76,7 +76,7 @@ func TestEnforceVersionPolicies(t *testing.T) {
 				kubeletVersion: "v1.12.3",
 				kubeadmVersion: "v1.12.3",
 			},
-			newK8sVersion:         "v1.11.10",
+			newK8sVersion:         "v1.10.10",
 			expectedMandatoryErrs: 1, // version must be higher than v1.12.0
 			expectedSkippableErrs: 1, // can't upgrade old k8s with newer kubeadm
 		},
@@ -85,9 +85,9 @@ func TestEnforceVersionPolicies(t *testing.T) {
 			vg: &fakeVersionGetter{
 				clusterVersion: "v1.11.3",
 				kubeletVersion: "v1.11.3",
-				kubeadmVersion: constants.CurrentKubernetesVersion.String(),
+				kubeadmVersion: "v1.13.0",
 			},
-			newK8sVersion:         constants.CurrentKubernetesVersion.String(),
+			newK8sVersion:         "v1.13.0",
 			expectedMandatoryErrs: 1, // can't upgrade two minor versions
 			expectedSkippableErrs: 1, // kubelet <-> apiserver skew too large
 		},
@@ -124,11 +124,11 @@ func TestEnforceVersionPolicies(t *testing.T) {
 		{
 			name: "the maximum skew between the cluster version and the kubelet versions should be one minor version. This may be forced through though.",
 			vg: &fakeVersionGetter{
-				clusterVersion: constants.MinimumControlPlaneVersion.WithPatch(3).String(),
-				kubeletVersion: "v1.12.8",
-				kubeadmVersion: constants.CurrentKubernetesVersion.String(),
+				clusterVersion: "v1.12.0",
+				kubeletVersion: "v1.10.8",
+				kubeadmVersion: "v1.12.0",
 			},
-			newK8sVersion:         constants.CurrentKubernetesVersion.String(),
+			newK8sVersion:         "v1.12.0",
 			expectedSkippableErrs: 1,
 		},
 		{

cmd/kubeadm/app/preflight/checks_test.go

@@ -793,7 +793,7 @@ func TestKubeletVersionCheck(t *testing.T) {
 		expectWarnings bool
 	}{
 		{"v" + constants.CurrentKubernetesVersion.WithPatch(2).String(), "", false, false},                                                                     // check minimally supported version when there is no information about control plane
-		{"v1.11.3", "v1.11.8", true, false},                                                                                                                    // too old kubelet (older than kubeadmconstants.MinimumKubeletVersion), should fail.
+		{"v1.1.0", "v1.11.8", true, false},                                                                                                                     // too old kubelet, should fail.
 		{"v" + constants.MinimumKubeletVersion.String(), constants.MinimumControlPlaneVersion.WithPatch(5).String(), false, false},                             // kubelet within same major.minor as control plane
 		{"v" + constants.MinimumKubeletVersion.WithPatch(5).String(), constants.MinimumControlPlaneVersion.WithPatch(1).String(), false, false},                // kubelet is newer, but still within same major.minor as control plane
 		{"v" + constants.MinimumKubeletVersion.String(), constants.CurrentKubernetesVersion.WithPatch(1).String(), false, false},                               // kubelet is lower than control plane, but newer than minimally supported

cmd/kubeadm/app/util/config/common.go

@@ -32,6 +32,8 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	netutil "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apimachinery/pkg/util/version"
+	apimachineryversion "k8s.io/apimachinery/pkg/version"
+	componentversion "k8s.io/component-base/version"
 	"k8s.io/klog/v2"
 
 	"github.com/pkg/errors"
@@ -102,8 +104,23 @@ func NormalizeKubernetesVersion(cfg *kubeadmapi.ClusterConfiguration) error {
 	if err != nil {
 		return errors.Wrapf(err, "couldn't parse Kubernetes version %q", cfg.KubernetesVersion)
 	}
-	if k8sVersion.LessThan(constants.MinimumControlPlaneVersion) {
-		return errors.Errorf("this version of kubeadm only supports deploying clusters with the control plane version >= %s. Current version: %s", constants.MinimumControlPlaneVersion.String(), cfg.KubernetesVersion)
+
+	// During the k8s release process, a kubeadm version in the main branch could be 1.23.0-pre,
+	// while the 1.22.0 version is not released yet. The MinimumControlPlaneVersion validation
+	// in such a case will not pass, since the value of MinimumControlPlaneVersion would be
+	// calculated as kubeadm version - 1 (1.22) and k8sVersion would still be at 1.21.x
+	// (fetched from the 'stable' marker). Handle this case by only showing a warning.
+	mcpVersion := constants.MinimumControlPlaneVersion
+	versionInfo := componentversion.Get()
+	if isKubeadmPrereleaseVersion(&versionInfo, k8sVersion, mcpVersion) {
+		klog.V(1).Infof("WARNING: tolerating control plane version %s, assuming that k8s version %s is not released yet",
+			cfg.KubernetesVersion, mcpVersion)
+		return nil
+	}
+	// If not a pre-release version, handle the validation normally.
+	if k8sVersion.LessThan(mcpVersion) {
+		return errors.Errorf("this version of kubeadm only supports deploying clusters with the control plane version >= %s. Current version: %s",
+			mcpVersion, cfg.KubernetesVersion)
 	}
 	return nil
 }
@@ -204,3 +221,20 @@ func MigrateOldConfig(oldConfig []byte) ([]byte, error) {
 
 	return bytes.Join(newConfig, []byte(constants.YAMLDocumentSeparator)), nil
 }
+
+// isKubeadmPrereleaseVersion returns true if the kubeadm version is a pre-release version and
+// the minimum control plane version is N+2 MINOR version of the given k8sVersion.
+func isKubeadmPrereleaseVersion(versionInfo *apimachineryversion.Info, k8sVersion, mcpVersion *version.Version) bool {
+	if len(versionInfo.Major) != 0 { // Make sure the component version is populated
+		kubeadmVersion := version.MustParseSemantic(versionInfo.String())
+		if len(kubeadmVersion.PreRelease()) != 0 { // Only handle this if the kubeadm binary is a pre-release
+			// After incrementing the k8s MINOR version by one, if this version is equal or greater than the
+			// MCP version, return true.
+			v := k8sVersion.WithMinor(k8sVersion.Minor() + 1)
+			if comp, _ := v.Compare(mcpVersion.String()); comp != -1 {
+				return true
+			}
+		}
+	}
+	return false
+}

cmd/kubeadm/app/util/config/common_test.go

@@ -26,6 +26,8 @@ import (
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/version"
+	apimachineryversion "k8s.io/apimachinery/pkg/version"
 
 	"github.com/lithammer/dedent"
 )
@@ -397,3 +399,55 @@ func TestMigrateOldConfigFromFile(t *testing.T) {
 		})
 	}
 }
+
+func TestIsKubeadmPrereleaseVersion(t *testing.T) {
+	validVersionInfo := &apimachineryversion.Info{Major: "1", GitVersion: "v1.23.0-alpha.1"}
+	tests := []struct {
+		name           string
+		versionInfo    *apimachineryversion.Info
+		k8sVersion     *version.Version
+		mcpVersion     *version.Version
+		expectedResult bool
+	}{
+		{
+			name:           "invalid versionInfo",
+			versionInfo:    &apimachineryversion.Info{},
+			expectedResult: false,
+		},
+		{
+			name:           "kubeadm is not a prerelease version",
+			versionInfo:    &apimachineryversion.Info{Major: "1", GitVersion: "v1.23.0"},
+			expectedResult: false,
+		},
+		{
+			name:           "mcpVersion is equal to k8sVersion",
+			versionInfo:    validVersionInfo,
+			k8sVersion:     version.MustParseSemantic("v1.21.0"),
+			mcpVersion:     version.MustParseSemantic("v1.21.0"),
+			expectedResult: true,
+		},
+		{
+			name:           "k8sVersion is 1 MINOR version older than mcpVersion",
+			versionInfo:    validVersionInfo,
+			k8sVersion:     version.MustParseSemantic("v1.21.0"),
+			mcpVersion:     version.MustParseSemantic("v1.22.0"),
+			expectedResult: true,
+		},
+		{
+			name:           "k8sVersion is 2 MINOR versions older than mcpVersion",
+			versionInfo:    validVersionInfo,
+			k8sVersion:     version.MustParseSemantic("v1.21.0"),
+			mcpVersion:     version.MustParseSemantic("v1.23.0"),
+			expectedResult: false,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			result := isKubeadmPrereleaseVersion(tc.versionInfo, tc.k8sVersion, tc.mcpVersion)
+			if result != tc.expectedResult {
+				t.Errorf("expected result: %v, got %v", tc.expectedResult, result)
+			}
+		})
+	}
+}

cmd/kubeadm/test/cmd/init_test.go

@@ -19,9 +19,10 @@ package kubeadm
 import (
 	"fmt"
 	"os"
+	"strings"
 	"testing"
 
-	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/apimachinery/pkg/util/version"
 
 	"github.com/lithammer/dedent"
 )
@@ -37,6 +38,16 @@ func runKubeadmInit(args ...string) (string, string, int, error) {
 	return RunCmd(kubeadmPath, kubeadmArgs...)
 }
 
+func getKubeadmVersion() *version.Version {
+	kubeadmPath := getKubeadmPath()
+	kubeadmArgs := []string{"version", "-o=short"}
+	out, _, _, err := RunCmd(kubeadmPath, kubeadmArgs...)
+	if err != nil {
+		panic(fmt.Sprintf("could not run 'kubeadm version -o=short': %v", err))
+	}
+	return version.MustParseSemantic(strings.TrimSpace(out))
+}
+
 func TestCmdInitToken(t *testing.T) {
 	initTest := []struct {
 		name     string
@@ -94,7 +105,7 @@ func TestCmdInitKubernetesVersion(t *testing.T) {
 		},
 		{
 			name:     "valid version is accepted",
-			args:     "--kubernetes-version=" + constants.CurrentKubernetesVersion.String(),
+			args:     "--kubernetes-version=" + getKubeadmVersion().String(),
 			expected: true,
 		},
 	}