wip

hack/tools/go.mod

@@ -8,6 +8,7 @@ replace sigs.k8s.io/cluster-api/test => ../../test
 
 require (
 	cloud.google.com/go/storage v1.36.0
+	github.com/Masterminds/sprig v2.22.0+incompatible
 	github.com/blang/semver/v4 v4.0.0
 	github.com/onsi/gomega v1.30.0
 	github.com/pkg/errors v0.9.1
@@ -37,6 +38,7 @@ require (
 	cloud.google.com/go/iam v1.1.5 // indirect
 	github.com/MakeNowJust/heredoc v1.0.0 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
+	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/Masterminds/semver/v3 v3.2.0 // indirect
 	github.com/Masterminds/sprig/v3 v3.2.3 // indirect
 	github.com/Microsoft/go-winio v0.5.2 // indirect

hack/tools/go.sum

@@ -16,8 +16,12 @@ github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ
 github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE=
 github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
 github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
+github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
+github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
 github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g=
 github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
+github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZCSYp4Z0m2dk6cEM60=
+github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
 github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA=
 github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM=
 github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA=

hack/tools/prowjob-generator/config.go

@@ -0,0 +1,42 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+// Config is the configuration file struct.
+type Config struct {
+	FileNamePrefix string                   `yaml:"fileNamePrefix"`
+	Branches       map[string]BranchConfig  `yaml:"branches"`
+	Versions       map[string]VersionMapper `yaml:"versions"`
+}
+
+// BranchConfig is the branch-based configuration struct.
+type BranchConfig struct {
+	Interval                            string    `yaml:"interval"`
+	KubekinsImage                       string    `yaml:"kubekinsImage"`
+	KubernetesVersionManagement         string    `yaml:"kubernetesVersionManagement"`
+	KubebuilderEnvtestKubernetesVersion string    `yaml:"kubebuilderEnvtestKubernetesVersion"`
+	Upgrades                            []Upgrade `yaml:"upgrades"`
+}
+
+// Upgrade describes a kubernetes upgrade.
+type Upgrade struct {
+	From string `yaml:"from"`
+	To   string `yaml:"to"`
+}
+
+// VersionMapper is a key value map strings to versions.
+type VersionMapper map[string]string

hack/tools/prowjob-generator/config.yaml

@@ -0,0 +1,116 @@
+fileNamePrefix: "cluster-api-"
+branches:
+  main:
+    kubekinsImage: "gcr.io/k8s-staging-test-infra/kubekins-e2e:v20231208-8b9fd88e88-1.29"
+    interval: "2h"
+    kubernetesVersionManagement: "v1.26.6@sha256:6e2d8b28a5b601defe327b98bd1c2d1930b49e5d8c512e1895099e4504007adb"
+    kubebuilderEnvtestKubernetesVersion: "1.26.1"
+    upgrades:
+    - from: "1.24"
+      to: "1.25"
+    - from: "1.25"
+      to: "1.26"
+    - from: "1.26"
+      to: "1.27"
+    - from: "1.27"
+      to: "1.28"
+    - from: "1.28"
+      to: "1.29"
+    - from: "1.29"
+      to: "latest"
+  release-1.6:
+    kubekinsImage: "gcr.io/k8s-staging-test-infra/kubekins-e2e:v20231206-f7b83ffbe6-1.28"
+    interval: "4h"
+    kubernetesVersionManagement: "v1.25.11@sha256:227fa11ce74ea76a0474eeefb84cb75d8dad1b08638371ecf0e86259b35be0c8"
+    kubebuilderEnvtestKubernetesVersion: "1.25.0"
+    upgrades:
+    - from: "1.23"
+      to: "1.24"
+    - from: "1.24"
+      to: "1.25"
+    - from: "1.25"
+      to: "1.26"
+    - from: "1.26"
+      to: "1.27"
+    - from: "1.27"
+      to: "1.28"
+    - from: "1.28"
+      to: "1.29"
+  release-1.5:
+    kubekinsImage: "gcr.io/k8s-staging-test-infra/kubekins-e2e:v20231206-f7b83ffbe6-1.27"
+    interval: "4h"
+    kubernetesVersionManagement: "v1.24.15"
+    kubebuilderEnvtestKubernetesVersion: "1.24.2"
+    upgrades:
+    - from: "1.22"
+      to: "1.23"
+    - from: "1.23"
+      to: "1.24"
+    - from: "1.24"
+      to: "1.25"
+    - from: "1.25"
+      to: "1.26"
+    - from: "1.26"
+      to: "1.27"
+    - from: "1.27"
+      to: "1.28"
+  release-1.4:
+    kubekinsImage: "gcr.io/k8s-staging-test-infra/kubekins-e2e:v20231206-f7b83ffbe6-1.26"
+    interval: "4h"
+    kubernetesVersionManagement: "v1.23.17"
+    kubebuilderEnvtestKubernetesVersion: "1.23.5"
+    upgrades:
+    - from: "1.21"
+      to: "1.22"
+    - from: "1.22"
+      to: "1.23"
+    - from: "1.23"
+      to: "1.24"
+    - from: "1.24"
+      to: "1.25"
+    - from: "1.25"
+      to: "1.26"
+    - from: "1.26"
+      to: "1.27"
+
+versions:
+  "1.21":
+    etcd: "3.5.3-0"
+    coreDNS: "v1.8.4"
+    resolvable: "stable-1.21"
+  "1.22":
+    etcd: "3.5.3-0"
+    coreDNS: "v1.8.6"
+    resolvable: "stable-1.22"
+  "1.23":
+    etcd: "3.5.3-0"
+    coreDNS: "v1.8.6"
+    resolvable: "stable-1.23"
+  "1.24":
+    etcd: "3.5.4-0"
+    coreDNS: "v1.9.3"
+    resolvable: "stable-1.24"
+  "1.25":
+    etcd: "3.5.6-0"
+    coreDNS: "v1.9.3"
+    resolvable: "stable-1.25"
+  "1.26":
+    etcd: "3.5.6-0"
+    coreDNS: "v1.9.3"
+    resolvable: "stable-1.26"
+  "1.27":
+    etcd: "3.5.9-0"
+    coreDNS: "v1.10.1"
+    resolvable: "stable-1.27"
+  "1.28":
+    etcd: "3.5.10-0"
+    coreDNS: "v1.11.1"
+    resolvable: "stable-1.28"
+  "1.29":
+    etcd: "3.5.10-0"
+    coreDNS: "v1.11.1"
+    resolvable: "stable-1.29"
+  "latest":
+    etcd: "3.5.10-0"
+    coreDNS: "v1.11.1"
+    resolvable: "ci/latest-1.30"

hack/tools/prowjob-generator/main.go

@@ -0,0 +1,179 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// main is the main package for prowjob-generator.
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"os"
+	"path"
+	"strings"
+	"text/template"
+
+	"github.com/Masterminds/sprig"
+	"github.com/pkg/errors"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/yaml"
+)
+
+var (
+	configFile = flag.String("config", "", "Path to the config file")
+	outputDir  = flag.String("output-dir", "", "Path to the directory to create the files in")
+
+	config Config
+)
+
+func init() {
+	config = Config{}
+}
+
+func main() {
+	flag.Parse()
+
+	if *outputDir == "" {
+		klog.Fatal("Expected flag \"output-dir\" to be set")
+	}
+
+	if *configFile == "" {
+		klog.Fatal("Expected flag \"config\" to be set")
+	}
+
+	rawConfig, err := os.ReadFile(*configFile)
+	if err != nil {
+		klog.Fatalf("Failed to read config file %q: %v", *configFile, err)
+	}
+
+	if err := yaml.Unmarshal(rawConfig, &config); err != nil {
+		klog.Fatalf("Failed to parse config file %q: %v", *configFile, err)
+	}
+
+	g, err := newGenerator(config, *outputDir)
+	if err != nil {
+		klog.Fatalf("Failed to initialize generator: %v", err)
+	}
+
+	if err := g.generate(); err != nil {
+		klog.Fatalf("Failed to generate prowjobs: %v", err)
+	}
+}
+
+func k8sShortVersionString(s string) string {
+	if strings.HasPrefix(s, "ci/latest-") {
+		return "latest"
+	}
+	return strings.ReplaceAll(strings.TrimPrefix(s, "stable-"), ".", "-")
+}
+
+func (g *generator) k8sVersionLookup(version, component string) string {
+	v, ok := g.config.Versions[version]
+	if !ok {
+		klog.Fatalf("Failed to lookup version (%q) in config", version)
+	}
+	c, ok := v[component]
+	if !ok {
+		klog.Fatalf("Failed to lookup component version (%q) for version %q in config", component, version)
+	}
+	return c
+}
+
+func (g *generator) lastUpgradeVersionFrom(branch string) string {
+	upgrades := g.config.Branches[branch].Upgrades
+	return upgrades[len(upgrades)-1].From
+}
+func (g *generator) lastUpgradeVersionTo(branch string) string {
+	upgrades := g.config.Branches[branch].Upgrades
+	return upgrades[len(upgrades)-1].To
+}
+
+func newGenerator(config Config, outputDir string) (*generator, error) {
+	g := &generator{
+		config:    config,
+		outputDir: outputDir,
+	}
+	funcs := sprig.HermeticTxtFuncMap()
+	funcs["k8sShortVersionString"] = k8sShortVersionString
+	funcs["k8sVersionLookup"] = g.k8sVersionLookup
+	funcs["lastUpgradeVersionFrom"] = g.lastUpgradeVersionFrom
+	funcs["lastUpgradeVersionTo"] = g.lastUpgradeVersionTo
+
+	var err error
+	g.templates, err = template.New("").Funcs(funcs).ParseFS(templatesFS, "templates/*.yaml")
+	if err != nil {
+		return nil, err
+	}
+
+	return g, err
+}
+
+type generator struct {
+	templates *template.Template
+	config    Config
+	outputDir string
+}
+
+var jobTypes = [][]string{
+	{"periodics"},
+	{"periodics", "upgrades"},
+	{"presubmits"},
+}
+
+func (g *generator) generate() error {
+	for _, jobType := range jobTypes {
+		for branch := range g.config.Branches {
+			if err := g.generateProwjobs(branch, jobType...); err != nil {
+				return errors.Wrapf(err, "Generating prowjobs for type %s", strings.Join(jobType, "-"))
+			}
+		}
+	}
+	return nil
+}
+
+func (g *generator) generateProwjobs(branch string, jobType ...string) error {
+	fileName := g.generateFilename(branch, jobType...)
+	templateName := strings.Join(jobType, "-") + ".yaml"
+
+	klog.Infof("Generating prowjobs to %q using template %q", fileName, templateName)
+
+	data := map[string]interface{}{
+		"branch": branch,
+		"config": g.config.Branches[branch],
+	}
+
+	var out bytes.Buffer
+	if err := g.templates.ExecuteTemplate(&out, templateName, data); err != nil {
+		return errors.Wrapf(err, "Executing template %q for branch %q", templateName, branch)
+	}
+
+	filePath := path.Join(g.outputDir, fileName)
+	if err := os.WriteFile(filePath, out.Bytes(), 0644); err != nil { //nolint:gosec
+		return errors.Wrapf(err, "Writing prowjob to %q", filePath)
+	}
+
+	return nil
+}
+
+func (g *generator) generateFilename(branch string, jobType ...string) string {
+	fileFriendlyBranchName := strings.ReplaceAll(branch, ".", "-")
+
+	n := fmt.Sprintf("%s%s-%s", g.config.FileNamePrefix, jobType[0], fileFriendlyBranchName)
+	if len(jobType) == 2 {
+		n = fmt.Sprintf("%s-%s", n, jobType[1])
+	}
+	return n + ".yaml"
+}

hack/tools/prowjob-generator/main_test.go

@@ -0,0 +1,30 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// main is the main package for prowjob-generator.
+package main
+
+import (
+	"testing"
+)
+
+func Test_newGenerator(t *testing.T) {
+	_, err := newGenerator(Config{}, "")
+	if err != nil {
+		t.Errorf("newGenerator() error = %v", err)
+		return
+	}
+}