From 8caa076a78550732c1e1d9edcd5ffd69b720943d Mon Sep 17 00:00:00 2001 From: tuti Date: Tue, 14 Jan 2025 09:33:24 -0800 Subject: [PATCH 1/7] allow creating a new operator based off a previous operator --- Makefile | 10 ++ go.mod | 10 +- go.sum | 9 +- hack/release-from/action.go | 263 ++++++++++++++++++++++++++++++++++++ hack/release-from/flags.go | 112 +++++++++++++++ hack/release-from/main.go | 57 ++++++++ hack/release-from/utils.go | 86 ++++++++++++ 7 files changed, 541 insertions(+), 6 deletions(-) create mode 100644 hack/release-from/action.go create mode 100644 hack/release-from/flags.go create mode 100644 hack/release-from/main.go create mode 100644 hack/release-from/utils.go diff --git a/Makefile b/Makefile index 19b86b60e2..9c86b82680 100644 --- a/Makefile +++ b/Makefile @@ -567,6 +567,7 @@ release-publish-images: release-prereqs release-check-image-exists release-github: hack/bin/gh release-notes @echo "Creating github release for $(VERSION)" hack/bin/gh release create $(VERSION) --title $(VERSION) --draft --notes-file $(VERSION)-release-notes.md + @echo "$(VERSION) GitHub release created in draft state. Please review and publish: https://github.com/tigera/operator/releases/tag/$(VERSION) ." GITHUB_CLI_VERSION?=2.62.0 hack/bin/gh: @@ -576,6 +577,15 @@ hack/bin/gh: chmod +x $@ rm hack/bin/gh.tgz +hack/bin/release-from: $(shell find ./hack/release-from -type f) + mkdir -p hack/bin + $(CONTAINERIZED) $(CALICO_BUILD) \ + sh -c '$(GIT_CONFIG_SSH) \ + go build -buildvcs=false -o hack/bin/release-from ./hack/release-from' + +release-from: hack/bin/release-from var-require-all-VERSION-OPERATOR_BASE_VERSION var-require-one-of-EE_IMAGES_VERSIONS-OS_IMAGES_VERSIONS + hack/bin/release-from + # release-prereqs checks that the environment is configured properly to create a release. release-prereqs: ifndef VERSION diff --git a/go.mod b/go.mod index 00945833b1..400817a8ba 100644 --- a/go.mod +++ b/go.mod @@ -49,7 +49,6 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/corazawaf/coraza-coreruleset/v4 v4.7.0 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/elastic/go-sysinfo v1.13.1 // indirect github.com/elastic/go-ucfg v0.8.8 // indirect @@ -117,16 +116,19 @@ require ( k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20240521193020-835d969ad83a // indirect k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 // indirect - sigs.k8s.io/gateway-api v1.1.0 // indirect + sigs.k8s.io/gateway-api v1.1.0 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) require ( - github.com/corazawaf/coraza-coreruleset v0.0.0-20240226094324-415b1017abdc // indirect - github.com/magefile/mage v1.14.0 // indirect + github.com/corazawaf/coraza-coreruleset v0.0.0-20240226094324-415b1017abdc + github.com/sirupsen/logrus v1.9.3 + github.com/urfave/cli/v3 v3.0.0-beta1 ) +require github.com/magefile/mage v1.14.0 // indirect + replace ( github.com/Azure/go-autorest => github.com/Azure/go-autorest v13.3.2+incompatible // Required by OLM github.com/imdario/mergo => github.com/imdario/mergo v0.3.16 // Per advice at https://github.com/darccio/mergo?tab=readme-ov-file#100 diff --git a/go.sum b/go.sum index 2bcf796112..f3c772851c 100644 --- a/go.sum +++ b/go.sum @@ -20,8 +20,6 @@ github.com/containernetworking/cni v1.2.3 h1:hhOcjNVUQTnzdRJ6alC5XF+wd9mfGIUaj8F github.com/containernetworking/cni v1.2.3/go.mod h1:DuLgF+aPd3DzcTQTtp/Nvl1Kim23oFKdm2okJzBQA5M= github.com/corazawaf/coraza-coreruleset v0.0.0-20240226094324-415b1017abdc h1:OlJhrgI3I+FLUCTI3JJW8MoqyM78WbqJjecqMnqG+wc= github.com/corazawaf/coraza-coreruleset v0.0.0-20240226094324-415b1017abdc/go.mod h1:7rsocqNDkTCira5T0M7buoKR2ehh7YZiPkzxRuAgvVU= -github.com/corazawaf/coraza-coreruleset/v4 v4.7.0 h1:j02CDxQYHVFZfBxbKLWYg66jSLbPmZp1GebyMwzN9Z0= -github.com/corazawaf/coraza-coreruleset/v4 v4.7.0/go.mod h1:1FQt1p+JSQ6tYrafMqZrEEdDmhq6aVuIJdnk+bM9hMY= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -209,6 +207,8 @@ github.com/r3labs/diff/v2 v2.15.1/go.mod h1:I8noH9Fc2fjSaMxqF3G2lhDdC0b+JXCfyx85 github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -219,10 +219,13 @@ github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tigera/api v0.0.0-20230406222214-ca74195900cb h1:Y7r5Al3V235KaEoAzGBz9RYXEbwDu8CPaZoCq2PlD8w= github.com/tigera/api v0.0.0-20230406222214-ca74195900cb/go.mod h1:ZZghiX3CUsBAc0osBjRvV6y/eun2ObYdvSbjqXAoj/w= +github.com/urfave/cli/v3 v3.0.0-beta1 h1:6DTaaUarcM0wX7qj5Hcvs+5Dm3dyUTBbEwIWAjcw9Zg= +github.com/urfave/cli/v3 v3.0.0-beta1/go.mod h1:FnIeEMYu+ko8zP1F9Ypr3xkZMIDqW3DR92yUtY39q1Y= github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= @@ -287,6 +290,7 @@ golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -342,6 +346,7 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= howett.net/plist v1.0.1 h1:37GdZ8tP09Q35o9ych3ehygcsL+HqKSwzctveSlarvM= diff --git a/hack/release-from/action.go b/hack/release-from/action.go new file mode 100644 index 0000000000..574bfde005 --- /dev/null +++ b/hack/release-from/action.go @@ -0,0 +1,263 @@ +// Copyright (c) 2025 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "regexp" + "strings" + + "github.com/sirupsen/logrus" + "github.com/urfave/cli/v3" + "gopkg.in/yaml.v2" +) + +type component struct { + Image string `yaml:"image"` + Version string `yaml:"version"` + Registry string `yaml:"registry"` +} + +type productConfig struct { + Title string `yaml:"title"` + Components map[string]component `yaml:"components"` +} + +func releaseFrom(ctx context.Context, c *cli.Command) error { + // get root directory of operator git repo + repoRootDir, err := runCommand("git", []string{"rev-parse", "--show-toplevel"}, nil) + if err != nil { + return fmt.Errorf("Error getting git root directory: %s", err) + } + + // fetch config from the base version of the operator + if err := retrieveBaseVersionConfig(baseOperatorFlag.Name, repoRootDir); err != nil { + return fmt.Errorf("Error getting base version config: %s", err) + } + + // Apply new version overrides + calicoOverrides := c.StringSlice(exceptCalicoFlag.Name) + if len(calicoOverrides) > 0 { + if err := updateConfig(repoRootDir, calicoConfig, calicoOverrides); err != nil { + return fmt.Errorf("Error overriding calico config: %s", err) + } + } + enterpriseOverrides := c.StringSlice(exceptEnterpriseFlag.Name) + if len(enterpriseOverrides) > 0 { + if err := updateConfig(repoRootDir, enterpriseConfig, enterpriseOverrides); err != nil { + return fmt.Errorf("Error overriding calico config: %s", err) + } + } + + // Either build a new release or a new hashrelease operator + version := c.String(versionFlag.Name) + release, err := isRelease(version) + if err != nil { + return fmt.Errorf("Error determining if version is a release: %s", err) + } else if release { + return newOperator(repoRootDir, version, c.String(remoteFlag.Name)) + } + + return newHashreleaseOperator(repoRootDir, version, c.StringSlice(archFlag.Name)) +} + +// isRelease checks if the version is a release version. +// A release version is in the format vX.Y.Z. +func isRelease(version string) (bool, error) { + releaseRegex, err := regexp.Compile(releaseFormat) + if err != nil { + return false, fmt.Errorf("Error compiling release regex: %s", err) + } + return releaseRegex.MatchString(version), nil +} + +// newOperator creates a new operator release. +func newOperator(dir, version, remote string) error { + // TODO: Commit, tag and push changes + if _, err := runCommandInDir(dir, "git", []string{"add", "config/"}, nil); err != nil { + return fmt.Errorf("Error adding changes in git: %s", err) + } + if _, err := runCommandInDir(dir, "git", []string{"commit", "-m", fmt.Sprintf("Release %s", version)}, nil); err != nil { + return fmt.Errorf("Error committing changes in git: %s", err) + } + if _, err := runCommandInDir(dir, "git", []string{"tag", version}, nil); err != nil { + return fmt.Errorf("Error tagging release in git: %s", err) + } + if _, err := runCommandInDir(dir, "git", []string{"push", remote, version}, nil); err != nil { + return fmt.Errorf("Error pushing tag in git: %s", err) + } + return nil +} + +// newHashreleaseOperator creates a new operator for a hashrelease. +func newHashreleaseOperator(dir, version string, archs []string) error { + env := os.Environ() + env = append(env, fmt.Sprintf("ARCHES=%s", strings.Join(archs, " "))) + env = append(env, fmt.Sprintf("GIT_VERSION=%s", version)) + if _, err := runCommandInDir(dir, "make", []string{"image-all"}, env); err != nil { + return err + } + for _, arch := range archs { + tag := fmt.Sprintf("%s/%s:%s-%s", quayRegistry, imageName, version, arch) + if _, err := runCommand("docker", []string{ + "tag", + fmt.Sprintf("%s:latest-%s", imageName, arch), + tag, + }, env); err != nil { + return err + } + logrus.WithField("tag", tag).Debug("Built image") + } + + initTag := fmt.Sprintf("%s/%s-init:%s", quayRegistry, imageName, version) + if _, err := runCommand("docker", []string{ + "tag", + fmt.Sprintf("%s-init:latest", imageName), + fmt.Sprintf("%s/%s-init:%s", quayRegistry, imageName, version), + }, env); err != nil { + return err + } + logrus.WithField("tag", initTag).Debug("Built init image") + return publishHashreleaseOperator(version, archs) +} + +// publishHashreleaseOperator publishes the hashrelease operator to quay. +func publishHashreleaseOperator(version string, archs []string) error { + multiArchTags := []string{} + for _, arch := range archs { + tag := fmt.Sprintf("%s/%s:%s-%s", quayRegistry, imageName, version, arch) + if _, err := runCommand("docker", []string{"push", tag}, nil); err != nil { + return err + } + logrus.WithField("tag", tag).Debug("Pushed image") + multiArchTags = append(multiArchTags, tag) + } + image := fmt.Sprintf("%s/%s:%s", quayRegistry, imageName, version) + cmd := []string{"manifest", "create", image} + for _, tag := range multiArchTags { + cmd = append(cmd, "--amend", tag) + } + if _, err := runCommand("docker", cmd, nil); err != nil { + return err + } + if _, err := runCommand("docker", []string{"manifest", "push", "--purge", image}, nil); err != nil { + return err + } + logrus.WithField("image", image).Debug("Pushed manifest") + + initImage := fmt.Sprintf("%s/%s-init:%s", quayRegistry, imageName, version) + if _, err := runCommand("docker", []string{"push", initImage}, nil); err != nil { + return err + } + logrus.WithField("image", initImage).Debug("Pushed init image") + return nil +} + +// updateConfig updates the version of image(s) in a config file +func updateConfig(repoRootDir, configFile string, updates []string) error { + components := make(map[string]string) + for _, override := range updates { + parts := strings.Split(override, ":") + if len(parts) != 2 { + return fmt.Errorf("Invalid override: %s", override) + } + components[parts[0]] = parts[1] + } + // open file locally + localFile := fmt.Sprintf("%s/%s", repoRootDir, configFile) + var config productConfig + if data, err := os.ReadFile(localFile); err != nil { + return fmt.Errorf("Error reading local file %s: %s", configFile, err) + } else if err = yaml.Unmarshal(data, &config); err != nil { + return fmt.Errorf("Error unmarshalling local file %s: %s", configFile, err) + } + for c, ver := range components { + if _, ok := config.Components[c]; ok { + config.Components[c] = component{ + Image: config.Components[c].Image, + Version: ver, + Registry: config.Components[c].Registry, + } + } + } + // overwrite local file with updated config + if err := os.WriteFile(localFile, []byte(fmt.Sprintf("%s\n", config)), 0o644); err != nil { + return fmt.Errorf("Error overwriting local file %s: %s", configFile, err) + } + return nil +} + +// retrieveBaseVersionConfig gets the config based to use +func retrieveBaseVersionConfig(baseVersion, repoRootDir string) error { + url, err := generateBaseConfigDownloadURL(baseVersion) + if err != nil { + return fmt.Errorf("Error getting download URL: %s", err) + } + + for _, file := range []string{calicoConfig, enterpriseConfig} { + // open file locally + localFile := fmt.Sprintf("%s/%s", repoRootDir, file) + out, err := os.OpenFile(localFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + return fmt.Errorf("Error opening local file %s: %s", file, err) + } + defer out.Close() + + // download file from base version + resp, err := http.Get(fmt.Sprintf("%s/%s", url, file)) + if err != nil { + return fmt.Errorf("Error downloading %s: %s", file, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("Error downloading %s: %s", file, resp.Status) + } + + // overwrite local file with downloaded file + if _, err = io.Copy(out, resp.Body); err != nil { + return fmt.Errorf("Error overwriting local file %s: %s", localFile, err) + } + logrus.WithFields(logrus.Fields{ + "file": file, + "localPath": localFile, + "downloadPath": url, + }).Debug("Overwrote local file with downloaded file") + } + return nil +} + +// generateBaseConfigDownloadURL returns the url to download base configs from +func generateBaseConfigDownloadURL(baseVersion string) (string, error) { + release, err := isRelease(baseVersion) + if err != nil { + return "", fmt.Errorf("Error determining if version is a release: %s", err) + } + if release { + return fmt.Sprintf("%s/refs/tags/%s", baseDownloadURL, baseVersion), nil + } + gitHashRegex, err := regexp.Compile(`^g([a-f0-9]{12})`) + if err != nil { + return "", fmt.Errorf("Error compiling git hash regex: %s", err) + } + matches := gitHashRegex.FindStringSubmatch(baseVersion) + if len(matches) < 1 { + return "", fmt.Errorf("Error finding git hash in base version") + } + return fmt.Sprintf("%s/blob/%s", baseDownloadURL, matches[1]), nil +} diff --git a/hack/release-from/flags.go b/hack/release-from/flags.go new file mode 100644 index 0000000000..1cae7fd699 --- /dev/null +++ b/hack/release-from/flags.go @@ -0,0 +1,112 @@ +// Copyright (c) 2025 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "regexp" + + "github.com/urfave/cli/v3" +) + +var debugFlag = &cli.BoolFlag{ + Name: "debug", + Usage: "Enable debug logging", + Sources: cli.EnvVars("DEBUG"), +} + +var remoteFlag = &cli.StringFlag{ + Name: "remote", + Usage: "The git remote to push the release to", + Value: "origin", + Sources: cli.EnvVars("GIT_REMOTE"), +} + +var baseOperatorFlag = &cli.StringFlag{ + Name: "base-version", + Aliases: []string{"base"}, + Usage: "The version of the operator to base this new version from. " + + "It is expected in the format vX.Y.Z for releases and " + + "for hashrelease, either vX.Y.Z-n-g- (legacy) or " + + "vX.Y.Z-n-g- (new) where product-hashrelease-version is in the format vA.B.C-u-g", + Sources: cli.EnvVars("OPERATOR_BASE_VERSION"), + Required: true, + Action: func(ctx context.Context, c *cli.Command, value string) error { + if !regexp.MustCompile(baseVersionFormat).MatchString(value) { + return fmt.Errorf("base-version must be in the format vX.Y.Z or vX.Y.Z-n-g- or " + + "vX.Y.Z-n-g-") + } + return nil + }, +} + +var versionFlag = &cli.StringFlag{ + Name: "version", + Usage: "The version of the operator to release", + Sources: cli.EnvVars("OPERATOR_VERSION", "VERSION"), + Action: func(ctx context.Context, c *cli.Command, value string) error { + if value == c.String("base-version") { + return fmt.Errorf("version cannot be the same as base-version") + } + if !regexp.MustCompile(baseVersionFormat).MatchString(value) { + return fmt.Errorf("base-version must be in the format vX.Y.Z or vX.Y.Z-n-g- or " + + "vX.Y.Z-n-g-") + } + return nil + }, +} + +var exceptCalicoFlag = &cli.StringSliceFlag{ + Name: "except-calico", + Usage: "A list of Calico images and the version to use for them. " + + "This should use the format based on the config/calico_versions.yaml file. " + + "e.g. --except-calico calico/cni:vX.Y.Z --except-calico csi-node-driver-registrar:vA.B.C-n-g", + Sources: cli.EnvVars("OS_IMAGES_VERSIONS"), +} + +var exceptEnterpriseFlag = &cli.StringSliceFlag{ + Name: "except-calico-enterprise", + Aliases: []string{"except-enterprise", "except-calient"}, + Usage: "A list of Enterprise images and the versions to use for them. " + + "This should use the format based on the config/enterprise_versions.yaml file. " + + "e.g. --except-calico-enterprise linseed:vX.Y.Z --except-calico-enterprise security-event-webhooks-processor:vA.B.C-n-g", + Sources: cli.EnvVars("EE_IMAGES_VERSIONS"), + Action: func(ctx context.Context, c *cli.Command, values []string) error { + if len(values) == 0 && len(c.StringSlice("except-calico")) == 0 { + return fmt.Errorf("at least one of --except-calico or --except-enterprise must be set") + } + return nil + }, +} + +var ( + archOptions = []string{"amd64", "arm64", "ppc64le", "s390x"} + archFlag = &cli.StringSliceFlag{ + Name: "architecture", + Aliases: []string{"arch"}, + Usage: "The architecture to use for the release. Repeat for multiple architectures.", + Sources: cli.EnvVars("ARCHS"), + Value: archOptions, + Action: func(ctx context.Context, c *cli.Command, values []string) error { + for _, arch := range values { + if !contains(archOptions, arch) { + return fmt.Errorf("invalid architecture %s", arch) + } + } + return nil + }, + } +) diff --git a/hack/release-from/main.go b/hack/release-from/main.go new file mode 100644 index 0000000000..399f35fc68 --- /dev/null +++ b/hack/release-from/main.go @@ -0,0 +1,57 @@ +// Copyright (c) 2025 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/sirupsen/logrus" + "github.com/urfave/cli/v3" +) + +func main() { + cmd := &cli.Command{ + Name: "operator-from", + Usage: "CLI tool for releasing operator using a previous release", + Flags: []cli.Flag{ + baseOperatorFlag, + versionFlag, + exceptCalicoFlag, + exceptEnterpriseFlag, + debugFlag, + }, + Before: func(ctx context.Context, c *cli.Command) (context.Context, error) { + if c.Bool(debugFlag.Name) { + logrus.SetLevel(logrus.DebugLevel) + } + // check if git repo is dirty + if version, err := gitVersion(); err != nil { + return ctx, fmt.Errorf("Error getting git version: %s", err) + } else if strings.Contains(version, "dirty") { + return ctx, fmt.Errorf("Git repo is dirty, please commit changes before releasing") + } + return ctx, nil + }, + Action: releaseFrom, + } + + // Run the app. + if err := cmd.Run(context.Background(), os.Args); err != nil { + logrus.WithError(err).Fatal("Error building new operator") + } +} diff --git a/hack/release-from/utils.go b/hack/release-from/utils.go new file mode 100644 index 0000000000..a50b39c20c --- /dev/null +++ b/hack/release-from/utils.go @@ -0,0 +1,86 @@ +// Copyright (c) 2025 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "fmt" + "io" + "os" + "os/exec" + "strings" + + "github.com/sirupsen/logrus" +) + +const ( + dockerHub = "docker.io" + quayRegistry = "quay.io" + + imageName = "tigera/operator" + + baseDownloadURL = "https://raw.githubusercontent.com/tigera/operator" + + calicoConfig = "config/calico_versions.yml" + enterpriseConfig = "config/enterprise_versions.yml" + + releaseFormat = `^v\d+\.\d+\.\d+$` + hashreleaseFormat = `^v\d+\.\d+\.\d+-\d+-g[a-f0-9]{12}-[a-z0-9-]+$` + baseVersionFormat = `^v\d+\.\d+\.\d+(-\d+-g[a-f0-9]{12}-[a-z0-9-]+)?$` +) + +func contains(haystack []string, needle string) bool { + for _, item := range haystack { + if item == needle { + return true + } + } + return false +} + +func gitVersion() (string, error) { + return runCommand("git", []string{"describe", "--tags", "--always", "--long", "--abbrev=12", "--dirty"}, nil) +} + +func runCommand(name string, args, env []string) (string, error) { + return runCommandInDir("", name, args, env) +} + +func runCommandInDir(dir, name string, args, env []string) (string, error) { + cmd := exec.Command(name, args...) + if len(env) != 0 { + cmd.Env = env + } + cmd.Dir = dir + var outb, errb bytes.Buffer + if logrus.IsLevelEnabled(logrus.DebugLevel) { + // If debug level is enabled, also write to stdout. + cmd.Stdout = io.MultiWriter(os.Stdout, &outb) + cmd.Stderr = io.MultiWriter(os.Stderr, &errb) + } else { + // Otherwise, just capture the output to return. + cmd.Stdout = io.MultiWriter(&outb) + cmd.Stderr = io.MultiWriter(&errb) + } + logrus.WithFields(logrus.Fields{ + "cmd": cmd.String(), + "dir": dir, + }).Debugf("Running %s command", name) + err := cmd.Run() + if err != nil { + err = fmt.Errorf("%s: %s", err, strings.TrimSpace(errb.String())) + } + return strings.TrimSpace(outb.String()), err +} From d70a81d959789e94a80a7073f17942d8f1c45170 Mon Sep 17 00:00:00 2001 From: tuti Date: Tue, 14 Jan 2025 12:07:02 -0800 Subject: [PATCH 2/7] address review feedback --- hack/release-from/README.md | 37 ++++++++++++++++++ hack/release-from/action.go | 76 +++++++++++++++++++++---------------- hack/release-from/flags.go | 7 ++++ 3 files changed, 87 insertions(+), 33 deletions(-) create mode 100644 hack/release-from/README.md diff --git a/hack/release-from/README.md b/hack/release-from/README.md new file mode 100644 index 0000000000..68a7f9891b --- /dev/null +++ b/hack/release-from/README.md @@ -0,0 +1,37 @@ +# release-from + +`release-from` is a tool designed to streamline the process of creating a new operator +using a previously released operator version. + +## Installation + +To install `release-from`, use the following command: + +```bash +make hack/bin/release-from +``` + +## Usage + +To start, familarize yourself with the tool + +```sh +release-from --help +``` + +To create a new release + +```sh +release-from --base-version --version \ + [--except-calico | --except-calico-enterprise] : +``` + +> [!IMPORTANT] +> To publish the newly created operator, use the `--publish` flag + +### Example + +```sh +release-from --base-version v1.36.2 --version v1.36.3 \ + --except-calico-enterprise linseed:v3.20.0-2.2 +``` diff --git a/hack/release-from/action.go b/hack/release-from/action.go index 574bfde005..7167a50b3e 100644 --- a/hack/release-from/action.go +++ b/hack/release-from/action.go @@ -39,6 +39,8 @@ type productConfig struct { Components map[string]component `yaml:"components"` } +// releaseFrom is the main action for the command. +// It builds (and publishes) a new operator based on the base version specified. func releaseFrom(ctx context.Context, c *cli.Command) error { // get root directory of operator git repo repoRootDir, err := runCommand("git", []string{"rev-parse", "--show-toplevel"}, nil) @@ -54,32 +56,31 @@ func releaseFrom(ctx context.Context, c *cli.Command) error { // Apply new version overrides calicoOverrides := c.StringSlice(exceptCalicoFlag.Name) if len(calicoOverrides) > 0 { - if err := updateConfig(repoRootDir, calicoConfig, calicoOverrides); err != nil { + if err := modifyComponentConfig(repoRootDir, calicoConfig, calicoOverrides); err != nil { return fmt.Errorf("Error overriding calico config: %s", err) } } enterpriseOverrides := c.StringSlice(exceptEnterpriseFlag.Name) if len(enterpriseOverrides) > 0 { - if err := updateConfig(repoRootDir, enterpriseConfig, enterpriseOverrides); err != nil { + if err := modifyComponentConfig(repoRootDir, enterpriseConfig, enterpriseOverrides); err != nil { return fmt.Errorf("Error overriding calico config: %s", err) } } - // Either build a new release or a new hashrelease operator + // Build either a new release or a new hashrelease operator version := c.String(versionFlag.Name) - release, err := isRelease(version) + isReleaseVersion, err := isReleaseVersionFormat(version) if err != nil { return fmt.Errorf("Error determining if version is a release: %s", err) - } else if release { - return newOperator(repoRootDir, version, c.String(remoteFlag.Name)) + } else if isReleaseVersion { + return newOperator(repoRootDir, version, c.String(remoteFlag.Name), c.Bool(publishFlag.Name)) } - return newHashreleaseOperator(repoRootDir, version, c.StringSlice(archFlag.Name)) + return newHashreleaseOperator(repoRootDir, version, c.StringSlice(archFlag.Name), c.Bool(publishFlag.Name)) } -// isRelease checks if the version is a release version. -// A release version is in the format vX.Y.Z. -func isRelease(version string) (bool, error) { +// isReleaseVersionFormat checks if the version in the format vX.Y.Z. +func isReleaseVersionFormat(version string) (bool, error) { releaseRegex, err := regexp.Compile(releaseFormat) if err != nil { return false, fmt.Errorf("Error compiling release regex: %s", err) @@ -88,8 +89,7 @@ func isRelease(version string) (bool, error) { } // newOperator creates a new operator release. -func newOperator(dir, version, remote string) error { - // TODO: Commit, tag and push changes +func newOperator(dir, version, remote string, publish bool) error { if _, err := runCommandInDir(dir, "git", []string{"add", "config/"}, nil); err != nil { return fmt.Errorf("Error adding changes in git: %s", err) } @@ -99,6 +99,10 @@ func newOperator(dir, version, remote string) error { if _, err := runCommandInDir(dir, "git", []string{"tag", version}, nil); err != nil { return fmt.Errorf("Error tagging release in git: %s", err) } + if !publish { + logrus.Info("skip pushing tag to git for publishing release") + return nil + } if _, err := runCommandInDir(dir, "git", []string{"push", remote, version}, nil); err != nil { return fmt.Errorf("Error pushing tag in git: %s", err) } @@ -106,14 +110,15 @@ func newOperator(dir, version, remote string) error { } // newHashreleaseOperator creates a new operator for a hashrelease. -func newHashreleaseOperator(dir, version string, archs []string) error { +// if publish is true, it will also publish the operator to registry. +func newHashreleaseOperator(dir, version string, arches []string, publish bool) error { env := os.Environ() - env = append(env, fmt.Sprintf("ARCHES=%s", strings.Join(archs, " "))) + env = append(env, fmt.Sprintf("ARCHES=%s", strings.Join(arches, " "))) env = append(env, fmt.Sprintf("GIT_VERSION=%s", version)) if _, err := runCommandInDir(dir, "make", []string{"image-all"}, env); err != nil { return err } - for _, arch := range archs { + for _, arch := range arches { tag := fmt.Sprintf("%s/%s:%s-%s", quayRegistry, imageName, version, arch) if _, err := runCommand("docker", []string{ "tag", @@ -134,10 +139,14 @@ func newHashreleaseOperator(dir, version string, archs []string) error { return err } logrus.WithField("tag", initTag).Debug("Built init image") - return publishHashreleaseOperator(version, archs) + if !publish { + logrus.Info("skip publishing images to registry") + return nil + } + return publishHashreleaseOperator(version, arches) } -// publishHashreleaseOperator publishes the hashrelease operator to quay. +// publishHashreleaseOperator publishes the hashrelease operator to registry. func publishHashreleaseOperator(version string, archs []string) error { multiArchTags := []string{} for _, arch := range archs { @@ -169,10 +178,10 @@ func publishHashreleaseOperator(version string, archs []string) error { return nil } -// updateConfig updates the version of image(s) in a config file -func updateConfig(repoRootDir, configFile string, updates []string) error { +// modifyComponentConfig updates the version of image(s) specified in the selected config file +func modifyComponentConfig(repoRootDir, configFile string, imgVerUpdates []string) error { components := make(map[string]string) - for _, override := range updates { + for _, override := range imgVerUpdates { parts := strings.Split(override, ":") if len(parts) != 2 { return fmt.Errorf("Invalid override: %s", override) @@ -203,24 +212,25 @@ func updateConfig(repoRootDir, configFile string, updates []string) error { return nil } -// retrieveBaseVersionConfig gets the config based to use +// retrieveBaseVersionConfig gets the config to use as a base for the new operator +// from the version specified as the base operator func retrieveBaseVersionConfig(baseVersion, repoRootDir string) error { - url, err := generateBaseConfigDownloadURL(baseVersion) + baseOperatorURL, err := generateBaseConfigDownloadURL(baseVersion) if err != nil { return fmt.Errorf("Error getting download URL: %s", err) } for _, file := range []string{calicoConfig, enterpriseConfig} { // open file locally - localFile := fmt.Sprintf("%s/%s", repoRootDir, file) - out, err := os.OpenFile(localFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + localFilePath := fmt.Sprintf("%s/%s", repoRootDir, file) + configFile, err := os.OpenFile(localFilePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) if err != nil { return fmt.Errorf("Error opening local file %s: %s", file, err) } - defer out.Close() + defer configFile.Close() // download file from base version - resp, err := http.Get(fmt.Sprintf("%s/%s", url, file)) + resp, err := http.Get(fmt.Sprintf("%s/%s", baseOperatorURL, file)) if err != nil { return fmt.Errorf("Error downloading %s: %s", file, err) } @@ -230,13 +240,13 @@ func retrieveBaseVersionConfig(baseVersion, repoRootDir string) error { } // overwrite local file with downloaded file - if _, err = io.Copy(out, resp.Body); err != nil { - return fmt.Errorf("Error overwriting local file %s: %s", localFile, err) + if _, err = io.Copy(configFile, resp.Body); err != nil { + return fmt.Errorf("Error overwriting local file %s: %s", localFilePath, err) } logrus.WithFields(logrus.Fields{ "file": file, - "localPath": localFile, - "downloadPath": url, + "localPath": localFilePath, + "downloadPath": baseOperatorURL, }).Debug("Overwrote local file with downloaded file") } return nil @@ -244,14 +254,14 @@ func retrieveBaseVersionConfig(baseVersion, repoRootDir string) error { // generateBaseConfigDownloadURL returns the url to download base configs from func generateBaseConfigDownloadURL(baseVersion string) (string, error) { - release, err := isRelease(baseVersion) + isReleaseVersion, err := isReleaseVersionFormat(baseVersion) if err != nil { return "", fmt.Errorf("Error determining if version is a release: %s", err) } - if release { + if isReleaseVersion { return fmt.Sprintf("%s/refs/tags/%s", baseDownloadURL, baseVersion), nil } - gitHashRegex, err := regexp.Compile(`^g([a-f0-9]{12})`) + gitHashRegex, err := regexp.Compile(`g([a-f0-9]{12})`) if err != nil { return "", fmt.Errorf("Error compiling git hash regex: %s", err) } diff --git a/hack/release-from/flags.go b/hack/release-from/flags.go index 1cae7fd699..7cc99917cc 100644 --- a/hack/release-from/flags.go +++ b/hack/release-from/flags.go @@ -110,3 +110,10 @@ var ( }, } ) + +var publishFlag = &cli.BoolFlag{ + Name: "publish", + Usage: "Publish the new operator", + Sources: cli.EnvVars("PUBLISH"), + Value: false, +} From 5c845741d9c91e4b22d258b6eb96f0bb2f727f92 Mon Sep 17 00:00:00 2001 From: tuti Date: Tue, 14 Jan 2025 13:11:34 -0800 Subject: [PATCH 3/7] additional changes - support custom registry - fixes from validating - modify config files and preserve comments - update examples in README --- hack/release-from/README.md | 25 ++++- hack/release-from/action.go | 210 +++++++++++++++++++++--------------- hack/release-from/flags.go | 68 +++++++++--- hack/release-from/main.go | 6 ++ hack/release-from/utils.go | 8 +- 5 files changed, 207 insertions(+), 110 deletions(-) diff --git a/hack/release-from/README.md b/hack/release-from/README.md index 68a7f9891b..4940e69c32 100644 --- a/hack/release-from/README.md +++ b/hack/release-from/README.md @@ -3,6 +3,8 @@ `release-from` is a tool designed to streamline the process of creating a new operator using a previously released operator version. +The base operator version must reference either a tag or commit hash in `tigera/operator` + ## Installation To install `release-from`, use the following command: @@ -29,9 +31,22 @@ release-from --base-version --version [!IMPORTANT] > To publish the newly created operator, use the `--publish` flag -### Example +### Examples -```sh -release-from --base-version v1.36.2 --version v1.36.3 \ - --except-calico-enterprise linseed:v3.20.0-2.2 -``` +1. To create a new operator with an updated `typha` for Calico to a custom registry + + ```sh + release-from --base-version v1.36.0-1.dev-259-g25c811f78fbd-v3.30.0-0.dev-338-gca80474016a5 --version v1.36.0-mod-typha \ + --except-calico typha:v3.30.0-0.dev-353-ge0bc56c0d646 --registry docker.io --image my-namespace/tigera-operator --publish + ``` + +1. To create a new operator release `v1.36.3` that has almost all the same images as `v1.36.2` + with the exception of Enterprise `linseed` component using `v3.20.0-2.2`. + + > [!WARNING] + > This assumes that user has push access to `tigera/operator` + + ```sh + release-from --base-version v1.36.2 --version v1.36.3 \ + --except-calico-enterprise linseed:v3.20.0-2.2 --publish + ``` diff --git a/hack/release-from/action.go b/hack/release-from/action.go index 7167a50b3e..fdd79e8d17 100644 --- a/hack/release-from/action.go +++ b/hack/release-from/action.go @@ -15,30 +15,18 @@ package main import ( + "bytes" "context" "fmt" - "io" - "net/http" "os" "regexp" "strings" "github.com/sirupsen/logrus" "github.com/urfave/cli/v3" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) -type component struct { - Image string `yaml:"image"` - Version string `yaml:"version"` - Registry string `yaml:"registry"` -} - -type productConfig struct { - Title string `yaml:"title"` - Components map[string]component `yaml:"components"` -} - // releaseFrom is the main action for the command. // It builds (and publishes) a new operator based on the base version specified. func releaseFrom(ctx context.Context, c *cli.Command) error { @@ -49,7 +37,7 @@ func releaseFrom(ctx context.Context, c *cli.Command) error { } // fetch config from the base version of the operator - if err := retrieveBaseVersionConfig(baseOperatorFlag.Name, repoRootDir); err != nil { + if err := retrieveBaseVersionConfig(c.String(baseOperatorFlag.Name), repoRootDir); err != nil { return fmt.Errorf("Error getting base version config: %s", err) } @@ -73,10 +61,10 @@ func releaseFrom(ctx context.Context, c *cli.Command) error { if err != nil { return fmt.Errorf("Error determining if version is a release: %s", err) } else if isReleaseVersion { - return newOperator(repoRootDir, version, c.String(remoteFlag.Name), c.Bool(publishFlag.Name)) + return newOperator(repoRootDir, version, c.String(gitRemoteFlag.Name), c.Bool(publishFlag.Name)) } - return newHashreleaseOperator(repoRootDir, version, c.StringSlice(archFlag.Name), c.Bool(publishFlag.Name)) + return newHashreleaseOperator(repoRootDir, version, c.String(imageFlag.Name), c.String(registryFlag.Name), c.StringSlice(archFlag.Name), c.Bool(publishFlag.Name)) } // isReleaseVersionFormat checks if the version in the format vX.Y.Z. @@ -88,7 +76,9 @@ func isReleaseVersionFormat(version string) (bool, error) { return releaseRegex.MatchString(version), nil } -// newOperator creates a new operator release. +// newOperator handles creating a new operator release. +// If publish is true, it will push a new tag to the git remote to trigger a release. +// Otherwise, it will only commit the changes to the git repo locally. func newOperator(dir, version, remote string, publish bool) error { if _, err := runCommandInDir(dir, "git", []string{"add", "config/"}, nil); err != nil { return fmt.Errorf("Error adding changes in git: %s", err) @@ -106,20 +96,35 @@ func newOperator(dir, version, remote string, publish bool) error { if _, err := runCommandInDir(dir, "git", []string{"push", remote, version}, nil); err != nil { return fmt.Errorf("Error pushing tag in git: %s", err) } + logrus.Warn("Ensure that the changes are merged into the main branch as well.") + logrus.Info("Follow the release progress in CI.") return nil } // newHashreleaseOperator creates a new operator for a hashrelease. // if publish is true, it will also publish the operator to registry. -func newHashreleaseOperator(dir, version string, arches []string, publish bool) error { +func newHashreleaseOperator(dir, version, imageName, registry string, arches []string, publish bool) error { + if err := buildHashreleaseOperator(dir, version, imageName, registry, arches); err != nil { + return fmt.Errorf("Error building operator: %s", err) + } + if !publish { + logrus.Info("skip publishing images to registry") + return nil + } + return publishHashreleaseOperator(version, imageName, registry, arches) +} + +func buildHashreleaseOperator(dir, version, imageName, registry string, arches []string) error { + initImageName := fmt.Sprintf("%s-init", imageName) env := os.Environ() env = append(env, fmt.Sprintf("ARCHES=%s", strings.Join(arches, " "))) env = append(env, fmt.Sprintf("GIT_VERSION=%s", version)) + env = append(env, fmt.Sprintf("BUILD_IMAGE=%s", imageName)) if _, err := runCommandInDir(dir, "make", []string{"image-all"}, env); err != nil { return err } for _, arch := range arches { - tag := fmt.Sprintf("%s/%s:%s-%s", quayRegistry, imageName, version, arch) + tag := fmt.Sprintf("%s/%s:%s-%s", registry, imageName, version, arch) if _, err := runCommand("docker", []string{ "tag", fmt.Sprintf("%s:latest-%s", imageName, arch), @@ -130,34 +135,39 @@ func newHashreleaseOperator(dir, version string, arches []string, publish bool) logrus.WithField("tag", tag).Debug("Built image") } - initTag := fmt.Sprintf("%s/%s-init:%s", quayRegistry, imageName, version) + env = os.Environ() + env = append(env, fmt.Sprintf("ARCHES=%s", strings.Join(arches, " "))) + env = append(env, fmt.Sprintf("GIT_VERSION=%s", version)) + env = append(env, fmt.Sprintf("BUILD_IMAGE=%s", imageName)) + env = append(env, fmt.Sprintf("BUILD_INIT_IMAGE=%s", initImageName)) + if _, err := runCommandInDir(dir, "make", []string{"image-init"}, env); err != nil { + return err + } + + initTag := fmt.Sprintf("%s/%s:%s", registry, initImageName, version) if _, err := runCommand("docker", []string{ "tag", - fmt.Sprintf("%s-init:latest", imageName), - fmt.Sprintf("%s/%s-init:%s", quayRegistry, imageName, version), + fmt.Sprintf("%s:latest", initImageName), + fmt.Sprintf("%s/%s:%s", registry, initImageName, version), }, env); err != nil { return err } logrus.WithField("tag", initTag).Debug("Built init image") - if !publish { - logrus.Info("skip publishing images to registry") - return nil - } - return publishHashreleaseOperator(version, arches) + return nil } // publishHashreleaseOperator publishes the hashrelease operator to registry. -func publishHashreleaseOperator(version string, archs []string) error { +func publishHashreleaseOperator(version, imageName, registry string, archs []string) error { multiArchTags := []string{} for _, arch := range archs { - tag := fmt.Sprintf("%s/%s:%s-%s", quayRegistry, imageName, version, arch) + tag := fmt.Sprintf("%s/%s:%s-%s", registry, imageName, version, arch) if _, err := runCommand("docker", []string{"push", tag}, nil); err != nil { return err } logrus.WithField("tag", tag).Debug("Pushed image") multiArchTags = append(multiArchTags, tag) } - image := fmt.Sprintf("%s/%s:%s", quayRegistry, imageName, version) + image := fmt.Sprintf("%s/%s:%s", registry, imageName, version) cmd := []string{"manifest", "create", image} for _, tag := range multiArchTags { cmd = append(cmd, "--amend", tag) @@ -170,7 +180,7 @@ func publishHashreleaseOperator(version string, archs []string) error { } logrus.WithField("image", image).Debug("Pushed manifest") - initImage := fmt.Sprintf("%s/%s-init:%s", quayRegistry, imageName, version) + initImage := fmt.Sprintf("%s/%s-init:%s", registry, imageName, version) if _, err := runCommand("docker", []string{"push", initImage}, nil); err != nil { return err } @@ -179,95 +189,117 @@ func publishHashreleaseOperator(version string, archs []string) error { } // modifyComponentConfig updates the version of image(s) specified in the selected config file -func modifyComponentConfig(repoRootDir, configFile string, imgVerUpdates []string) error { - components := make(map[string]string) - for _, override := range imgVerUpdates { - parts := strings.Split(override, ":") - if len(parts) != 2 { - return fmt.Errorf("Invalid override: %s", override) - } - components[parts[0]] = parts[1] - } +func modifyComponentConfig(repoRootDir, configFile string, updates []string) error { // open file locally - localFile := fmt.Sprintf("%s/%s", repoRootDir, configFile) - var config productConfig - if data, err := os.ReadFile(localFile); err != nil { + localFilePath := fmt.Sprintf("%s/%s", repoRootDir, configFile) + var root yaml.Node + if data, err := os.ReadFile(localFilePath); err != nil { return fmt.Errorf("Error reading local file %s: %s", configFile, err) - } else if err = yaml.Unmarshal(data, &config); err != nil { + } else if err = yaml.Unmarshal(data, &root); err != nil { return fmt.Errorf("Error unmarshalling local file %s: %s", configFile, err) } - for c, ver := range components { - if _, ok := config.Components[c]; ok { - config.Components[c] = component{ - Image: config.Components[c].Image, - Version: ver, - Registry: config.Components[c].Registry, - } + + for _, override := range updates { + parts := strings.Split(override, ":") + component := parts[0] + version := parts[1] + if err := updateComponentVersion(&root, []string{"components", component, "version"}, version); err != nil { + return fmt.Errorf("Error updating component %s to %s: %s", component, version, err) } } + // overwrite local file with updated config - if err := os.WriteFile(localFile, []byte(fmt.Sprintf("%s\n", config)), 0o644); err != nil { + var buf bytes.Buffer + encoder := yaml.NewEncoder(&buf) + encoder.SetIndent(2) + if err := encoder.Encode(&root); err != nil { + return fmt.Errorf("Error encoding updated config: %s", err) + } + encoder.Close() + if err := os.WriteFile(localFilePath, buf.Bytes(), 0o644); err != nil { return fmt.Errorf("Error overwriting local file %s: %s", configFile, err) } return nil } -// retrieveBaseVersionConfig gets the config to use as a base for the new operator -// from the version specified as the base operator -func retrieveBaseVersionConfig(baseVersion, repoRootDir string) error { - baseOperatorURL, err := generateBaseConfigDownloadURL(baseVersion) - if err != nil { - return fmt.Errorf("Error getting download URL: %s", err) - } +// updateComponentVersion traverses the yaml node to update the version of the component. +func updateComponentVersion(node *yaml.Node, path []string, version string) error { + current := node.Content[0] + for i, key := range path { + found := false + for j := 0; j < len(current.Content)-1; j += 2 { + keyNode := current.Content[j] + valueNode := current.Content[j+1] - for _, file := range []string{calicoConfig, enterpriseConfig} { - // open file locally - localFilePath := fmt.Sprintf("%s/%s", repoRootDir, file) - configFile, err := os.OpenFile(localFilePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) - if err != nil { - return fmt.Errorf("Error opening local file %s: %s", file, err) - } - defer configFile.Close() + logrus.WithFields(logrus.Fields{ + "key": keyNode.Value, + "value": valueNode.Value, + }).Debug("Checking key and value") - // download file from base version - resp, err := http.Get(fmt.Sprintf("%s/%s", baseOperatorURL, file)) - if err != nil { - return fmt.Errorf("Error downloading %s: %s", file, err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("Error downloading %s: %s", file, resp.Status) + if keyNode.Value == key { + if i == len(path)-1 { + valueNode.Value = version + return nil + } + + if valueNode.Kind == yaml.MappingNode { + current = valueNode + found = true + break + } else { + return fmt.Errorf("expected mapping node at path %v, got %v", path[:i+1], valueNode.Kind) + } + } } - // overwrite local file with downloaded file - if _, err = io.Copy(configFile, resp.Body); err != nil { - return fmt.Errorf("Error overwriting local file %s: %s", localFilePath, err) + if !found { + return fmt.Errorf("key '%s' not found at path %v", key, path[:i+1]) } + } + return nil +} + +// retrieveBaseVersionConfig gets the config to use as a base for the new operator +// from the base version of the operator. +func retrieveBaseVersionConfig(baseVersion, repoRootDir string) error { + gitHashOrTag, err := extractGitHashOrTag(baseVersion) + if err != nil { + return fmt.Errorf("Error extracting git hash or tag from %q: %s", baseVersion, err) + } + + for _, configFilePath := range []string{calicoConfig, enterpriseConfig} { + localFilePath := fmt.Sprintf("%s/%s", repoRootDir, configFilePath) + url := fmt.Sprintf(sourceGitHubURL, gitHashOrTag, configFilePath) logrus.WithFields(logrus.Fields{ - "file": file, + "file": configFilePath, "localPath": localFilePath, - "downloadPath": baseOperatorURL, - }).Debug("Overwrote local file with downloaded file") + "downloadPath": url, + }).Debug("Replacing local file with downloaded file") + + if _, err := runCommand("curl", []string{"-L", "-o", localFilePath, url}, nil); err != nil { + return fmt.Errorf("Error downloading %s from %s: %s", configFilePath, url, err) + } } return nil } -// generateBaseConfigDownloadURL returns the url to download base configs from -func generateBaseConfigDownloadURL(baseVersion string) (string, error) { +// extractGitHashOrTag returns the tag for a release version +// or the git hash for a hashrelease version from the baseVersion. +func extractGitHashOrTag(baseVersion string) (string, error) { isReleaseVersion, err := isReleaseVersionFormat(baseVersion) if err != nil { return "", fmt.Errorf("Error determining if version is a release: %s", err) } if isReleaseVersion { - return fmt.Sprintf("%s/refs/tags/%s", baseDownloadURL, baseVersion), nil + return baseVersion, nil } - gitHashRegex, err := regexp.Compile(`g([a-f0-9]{12})`) + gitHashRegex, err := regexp.Compile(`g([0-9a-f]{12})`) if err != nil { return "", fmt.Errorf("Error compiling git hash regex: %s", err) } matches := gitHashRegex.FindStringSubmatch(baseVersion) - if len(matches) < 1 { - return "", fmt.Errorf("Error finding git hash in base version") + if len(matches) > 1 { + return matches[1], nil } - return fmt.Sprintf("%s/blob/%s", baseDownloadURL, matches[1]), nil + return "", fmt.Errorf("Error finding git hash in base version") } diff --git a/hack/release-from/flags.go b/hack/release-from/flags.go index 7cc99917cc..8ad956b250 100644 --- a/hack/release-from/flags.go +++ b/hack/release-from/flags.go @@ -18,7 +18,9 @@ import ( "context" "fmt" "regexp" + "strings" + "github.com/sirupsen/logrus" "github.com/urfave/cli/v3" ) @@ -28,7 +30,7 @@ var debugFlag = &cli.BoolFlag{ Sources: cli.EnvVars("DEBUG"), } -var remoteFlag = &cli.StringFlag{ +var gitRemoteFlag = &cli.StringFlag{ Name: "remote", Usage: "The git remote to push the release to", Value: "origin", @@ -41,29 +43,44 @@ var baseOperatorFlag = &cli.StringFlag{ Usage: "The version of the operator to base this new version from. " + "It is expected in the format vX.Y.Z for releases and " + "for hashrelease, either vX.Y.Z-n-g- (legacy) or " + - "vX.Y.Z-n-g- (new) where product-hashrelease-version is in the format vA.B.C-u-g", + "vX.Y.Z--n-g- (new) " + + "where product-hashrelease-version is in the format vA.B.C--u-g.", Sources: cli.EnvVars("OPERATOR_BASE_VERSION"), Required: true, Action: func(ctx context.Context, c *cli.Command, value string) error { - if !regexp.MustCompile(baseVersionFormat).MatchString(value) { - return fmt.Errorf("base-version must be in the format vX.Y.Z or vX.Y.Z-n-g- or " + - "vX.Y.Z-n-g-") + if !regexp.MustCompile(fmt.Sprintf(baseVersionFormat, c.String(devTagSuffixFlag.Name))).MatchString(value) { + return fmt.Errorf("base-version must be in the format vX.Y.Z or vX.Y.Z--n-g- or " + + "vX.Y.Z--n-g-") } return nil }, } +var devTagSuffixFlag = &cli.StringFlag{ + Name: "dev-tag-suffix", + Usage: "The suffix used to denote development tags", + Sources: cli.EnvVars("DEV_TAG_SUFFIX"), + Value: "0-dev", +} + var versionFlag = &cli.StringFlag{ - Name: "version", - Usage: "The version of the operator to release", - Sources: cli.EnvVars("OPERATOR_VERSION", "VERSION"), + Name: "version", + Usage: "The version of the operator to release", + Sources: cli.EnvVars("OPERATOR_VERSION", "VERSION"), + Required: true, Action: func(ctx context.Context, c *cli.Command, value string) error { if value == c.String("base-version") { return fmt.Errorf("version cannot be the same as base-version") } - if !regexp.MustCompile(baseVersionFormat).MatchString(value) { - return fmt.Errorf("base-version must be in the format vX.Y.Z or vX.Y.Z-n-g- or " + - "vX.Y.Z-n-g-") + if regexp.MustCompile(releaseFormat).MatchString(value) { + logrus.Warn("You are releasing a new operator version.") + return nil + } + if !regexp.MustCompile(fmt.Sprintf(hashreleaseFormat, c.String(devTagSuffixFlag.Name))).MatchString(value) { + if c.Bool(publishFlag.Name) && c.String(registryFlag.Name) == quayRegistry && c.String(imageFlag.Name) == defaultImageName { + return fmt.Errorf("cannot use the default registry and image for publishing operator version %q. "+ + "Either update registry and/or image flag OR specify version in the format ", value) + } } return nil }, @@ -75,6 +92,7 @@ var exceptCalicoFlag = &cli.StringSliceFlag{ "This should use the format based on the config/calico_versions.yaml file. " + "e.g. --except-calico calico/cni:vX.Y.Z --except-calico csi-node-driver-registrar:vA.B.C-n-g", Sources: cli.EnvVars("OS_IMAGES_VERSIONS"), + Action: validateOverrides, } var exceptEnterpriseFlag = &cli.StringSliceFlag{ @@ -88,10 +106,20 @@ var exceptEnterpriseFlag = &cli.StringSliceFlag{ if len(values) == 0 && len(c.StringSlice("except-calico")) == 0 { return fmt.Errorf("at least one of --except-calico or --except-enterprise must be set") } - return nil + return validateOverrides(ctx, c, values) }, } +func validateOverrides(ctx context.Context, c *cli.Command, values []string) error { + for _, value := range values { + parts := strings.Split(value, ":") + if len(parts) != 2 { + return fmt.Errorf("invalid override %q, must be in the format :", value) + } + } + return nil +} + var ( archOptions = []string{"amd64", "arm64", "ppc64le", "s390x"} archFlag = &cli.StringSliceFlag{ @@ -111,6 +139,22 @@ var ( } ) +var registryFlag = &cli.StringFlag{ + Name: "registry", + Usage: "The registry to push the new operator to. " + + "This is only used when creating a new operator for hashreleases.", + Sources: cli.EnvVars("REGISTRY"), + Value: quayRegistry, +} + +var imageFlag = &cli.StringFlag{ + Name: "image", + Usage: "The image name to use for the new operator. " + + "This is only used when creating a new operator for hashreleases.", + Sources: cli.EnvVars("IMAGE_NAME"), + Value: defaultImageName, +} + var publishFlag = &cli.BoolFlag{ Name: "publish", Usage: "Publish the new operator", diff --git a/hack/release-from/main.go b/hack/release-from/main.go index 399f35fc68..720123ae9b 100644 --- a/hack/release-from/main.go +++ b/hack/release-from/main.go @@ -33,6 +33,12 @@ func main() { versionFlag, exceptCalicoFlag, exceptEnterpriseFlag, + publishFlag, + archFlag, + gitRemoteFlag, + registryFlag, + imageFlag, + devTagSuffixFlag, debugFlag, }, Before: func(ctx context.Context, c *cli.Command) (context.Context, error) { diff --git a/hack/release-from/utils.go b/hack/release-from/utils.go index a50b39c20c..2e293a9439 100644 --- a/hack/release-from/utils.go +++ b/hack/release-from/utils.go @@ -29,16 +29,16 @@ const ( dockerHub = "docker.io" quayRegistry = "quay.io" - imageName = "tigera/operator" + defaultImageName = "tigera/operator" - baseDownloadURL = "https://raw.githubusercontent.com/tigera/operator" + sourceGitHubURL = `https://github.com/tigera/operator/raw/%s/%s` calicoConfig = "config/calico_versions.yml" enterpriseConfig = "config/enterprise_versions.yml" releaseFormat = `^v\d+\.\d+\.\d+$` - hashreleaseFormat = `^v\d+\.\d+\.\d+-\d+-g[a-f0-9]{12}-[a-z0-9-]+$` - baseVersionFormat = `^v\d+\.\d+\.\d+(-\d+-g[a-f0-9]{12}-[a-z0-9-]+)?$` + hashreleaseFormat = `^v\d+\.\d+\.\d+-%s-\d+-g[a-f0-9]{12}-[a-z0-9-]+$` + baseVersionFormat = `^v\d+\.\d+\.\d+(-%s-\d+-g[a-f0-9]{12}-[a-z0-9-]+)?$` ) func contains(haystack []string, needle string) bool { From e2badcb28d20b5dd32232903ea0915782ef349ad Mon Sep 17 00:00:00 2001 From: tuti Date: Thu, 16 Jan 2025 07:20:41 -0800 Subject: [PATCH 4/7] update go.mod --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 400817a8ba..9bfae90cc7 100644 --- a/go.mod +++ b/go.mod @@ -111,7 +111,7 @@ require ( google.golang.org/protobuf v1.34.2 // indirect gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + gopkg.in/yaml.v3 v3.0.1 howett.net/plist v1.0.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20240521193020-835d969ad83a // indirect From be41c55682f65d6db6f19d090ac7406cd2fe2c15 Mon Sep 17 00:00:00 2001 From: tuti Date: Thu, 23 Jan 2025 14:09:25 -0800 Subject: [PATCH 5/7] address review feedback --- go.mod | 14 ++++------- hack/release-from/README.md | 26 +++++++++++++++++++-- hack/release-from/action.go | 46 +++++++++++++++++++------------------ hack/release-from/flags.go | 33 +++++++++----------------- hack/release-from/main.go | 4 ++-- 5 files changed, 66 insertions(+), 57 deletions(-) diff --git a/go.mod b/go.mod index 28c87f8fd0..92fd6979a8 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/aws/aws-sdk-go v1.55.5 github.com/cloudflare/cfssl v1.6.5 github.com/containernetworking/cni v1.2.3 + github.com/corazawaf/coraza-coreruleset v0.0.0-20240226094324-415b1017abdc github.com/elastic/cloud-on-k8s/v2 v2.14.0 github.com/envoyproxy/gateway v1.1.2 github.com/go-ldap/ldap v3.0.3+incompatible @@ -24,13 +25,16 @@ require ( github.com/projectcalico/api v0.0.0-20240708202104-e3f70b269c2c github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.74.0 github.com/r3labs/diff/v2 v2.15.1 + github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.9.0 github.com/tigera/api v0.0.0-20230406222214-ca74195900cb + github.com/urfave/cli/v3 v3.0.0-beta1 go.uber.org/zap v1.27.0 golang.org/x/crypto v0.28.0 golang.org/x/net v0.30.0 gopkg.in/inf.v0 v0.9.1 gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.30.5 k8s.io/apiextensions-apiserver v0.30.5 k8s.io/apimachinery v0.30.5 @@ -80,6 +84,7 @@ require ( github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/magefile/mage v1.14.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -112,7 +117,6 @@ require ( google.golang.org/protobuf v1.34.2 // indirect gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect - gopkg.in/yaml.v3 v3.0.1 howett.net/plist v1.0.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20240521193020-835d969ad83a // indirect @@ -121,14 +125,6 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) -require ( - github.com/corazawaf/coraza-coreruleset v0.0.0-20240226094324-415b1017abdc - github.com/sirupsen/logrus v1.9.3 - github.com/urfave/cli/v3 v3.0.0-beta1 -) - -require github.com/magefile/mage v1.14.0 // indirect - replace ( github.com/Azure/go-autorest => github.com/Azure/go-autorest v13.3.2+incompatible // Required by OLM github.com/imdario/mergo => github.com/imdario/mergo v0.3.16 // Per advice at https://github.com/darccio/mergo?tab=readme-ov-file#100 diff --git a/hack/release-from/README.md b/hack/release-from/README.md index 4940e69c32..dbdc5c88e3 100644 --- a/hack/release-from/README.md +++ b/hack/release-from/README.md @@ -21,7 +21,7 @@ To start, familarize yourself with the tool release-from --help ``` -To create a new release +To create a new release. See [Examples](#examples) below for usage options ```sh release-from --base-version --version \ @@ -30,9 +30,19 @@ release-from --base-version --version [!IMPORTANT] > To publish the newly created operator, use the `--publish` flag +> +> `--publish` will push the operator image to remote repository +> and ONLY create a draft release on the [Releases](https://github.com/tigera/operator/releases) page for release versions (i.e. vX.Y.Z) ### Examples +1. To create a new operator with an updated `typha` for Calico to a custom registry locally + + ```sh + release-from --base-version v1.36.0-1.dev-259-g25c811f78fbd-v3.30.0-0.dev-338-gca80474016a5 --version v1.36.0-mod-typha \ + --except-calico typha:v3.30.0-0.dev-353-ge0bc56c0d646 --registry docker.io --image my-namespace/tigera-operator + ``` + 1. To create a new operator with an updated `typha` for Calico to a custom registry ```sh @@ -40,11 +50,23 @@ release-from --base-version --version [!WARNING] + > This assumes that user has push access to [`tigera/operator`](https://github.com/tigera/operator) + + ```sh + release-from --base-version v1.36.2 --version v1.36.3 \ + --except-calico-enterprise linseed:v3.20.0-2.2 + ``` + 1. To create a new operator release `v1.36.3` that has almost all the same images as `v1.36.2` with the exception of Enterprise `linseed` component using `v3.20.0-2.2`. > [!WARNING] - > This assumes that user has push access to `tigera/operator` + > This assumes that user has push access to [`tigera/operator`](https://github.com/tigera/operator) + > and [`quay.io/tigera/operator`](https://quay.io/repository/tigera/operator) ```sh release-from --base-version v1.36.2 --version v1.36.3 \ diff --git a/hack/release-from/action.go b/hack/release-from/action.go index fdd79e8d17..1122570fde 100644 --- a/hack/release-from/action.go +++ b/hack/release-from/action.go @@ -33,25 +33,25 @@ func releaseFrom(ctx context.Context, c *cli.Command) error { // get root directory of operator git repo repoRootDir, err := runCommand("git", []string{"rev-parse", "--show-toplevel"}, nil) if err != nil { - return fmt.Errorf("Error getting git root directory: %s", err) + return fmt.Errorf("error getting git root directory: %s", err) } // fetch config from the base version of the operator if err := retrieveBaseVersionConfig(c.String(baseOperatorFlag.Name), repoRootDir); err != nil { - return fmt.Errorf("Error getting base version config: %s", err) + return fmt.Errorf("error getting base version config: %s", err) } // Apply new version overrides calicoOverrides := c.StringSlice(exceptCalicoFlag.Name) if len(calicoOverrides) > 0 { if err := modifyComponentConfig(repoRootDir, calicoConfig, calicoOverrides); err != nil { - return fmt.Errorf("Error overriding calico config: %s", err) + return fmt.Errorf("error overriding calico config: %s", err) } } enterpriseOverrides := c.StringSlice(exceptEnterpriseFlag.Name) if len(enterpriseOverrides) > 0 { if err := modifyComponentConfig(repoRootDir, enterpriseConfig, enterpriseOverrides); err != nil { - return fmt.Errorf("Error overriding calico config: %s", err) + return fmt.Errorf("error overriding calico config: %s", err) } } @@ -59,7 +59,7 @@ func releaseFrom(ctx context.Context, c *cli.Command) error { version := c.String(versionFlag.Name) isReleaseVersion, err := isReleaseVersionFormat(version) if err != nil { - return fmt.Errorf("Error determining if version is a release: %s", err) + return fmt.Errorf("error determining if version is a release: %s", err) } else if isReleaseVersion { return newOperator(repoRootDir, version, c.String(gitRemoteFlag.Name), c.Bool(publishFlag.Name)) } @@ -71,7 +71,7 @@ func releaseFrom(ctx context.Context, c *cli.Command) error { func isReleaseVersionFormat(version string) (bool, error) { releaseRegex, err := regexp.Compile(releaseFormat) if err != nil { - return false, fmt.Errorf("Error compiling release regex: %s", err) + return false, fmt.Errorf("error compiling release regex: %s", err) } return releaseRegex.MatchString(version), nil } @@ -81,20 +81,20 @@ func isReleaseVersionFormat(version string) (bool, error) { // Otherwise, it will only commit the changes to the git repo locally. func newOperator(dir, version, remote string, publish bool) error { if _, err := runCommandInDir(dir, "git", []string{"add", "config/"}, nil); err != nil { - return fmt.Errorf("Error adding changes in git: %s", err) + return fmt.Errorf("error adding changes in git: %s", err) } if _, err := runCommandInDir(dir, "git", []string{"commit", "-m", fmt.Sprintf("Release %s", version)}, nil); err != nil { - return fmt.Errorf("Error committing changes in git: %s", err) + return fmt.Errorf("error committing changes in git: %s", err) } if _, err := runCommandInDir(dir, "git", []string{"tag", version}, nil); err != nil { - return fmt.Errorf("Error tagging release in git: %s", err) + return fmt.Errorf("error tagging release in git: %s", err) } if !publish { logrus.Info("skip pushing tag to git for publishing release") return nil } if _, err := runCommandInDir(dir, "git", []string{"push", remote, version}, nil); err != nil { - return fmt.Errorf("Error pushing tag in git: %s", err) + return fmt.Errorf("error pushing tag in git: %s", err) } logrus.Warn("Ensure that the changes are merged into the main branch as well.") logrus.Info("Follow the release progress in CI.") @@ -105,7 +105,7 @@ func newOperator(dir, version, remote string, publish bool) error { // if publish is true, it will also publish the operator to registry. func newHashreleaseOperator(dir, version, imageName, registry string, arches []string, publish bool) error { if err := buildHashreleaseOperator(dir, version, imageName, registry, arches); err != nil { - return fmt.Errorf("Error building operator: %s", err) + return fmt.Errorf("error building operator: %s", err) } if !publish { logrus.Info("skip publishing images to registry") @@ -194,9 +194,9 @@ func modifyComponentConfig(repoRootDir, configFile string, updates []string) err localFilePath := fmt.Sprintf("%s/%s", repoRootDir, configFile) var root yaml.Node if data, err := os.ReadFile(localFilePath); err != nil { - return fmt.Errorf("Error reading local file %s: %s", configFile, err) + return fmt.Errorf("error reading local file %s: %s", configFile, err) } else if err = yaml.Unmarshal(data, &root); err != nil { - return fmt.Errorf("Error unmarshalling local file %s: %s", configFile, err) + return fmt.Errorf("error unmarshalling local file %s: %s", configFile, err) } for _, override := range updates { @@ -204,7 +204,7 @@ func modifyComponentConfig(repoRootDir, configFile string, updates []string) err component := parts[0] version := parts[1] if err := updateComponentVersion(&root, []string{"components", component, "version"}, version); err != nil { - return fmt.Errorf("Error updating component %s to %s: %s", component, version, err) + return fmt.Errorf("error updating component %s to %s: %s", component, version, err) } } @@ -213,11 +213,13 @@ func modifyComponentConfig(repoRootDir, configFile string, updates []string) err encoder := yaml.NewEncoder(&buf) encoder.SetIndent(2) if err := encoder.Encode(&root); err != nil { - return fmt.Errorf("Error encoding updated config: %s", err) + return fmt.Errorf("error encoding updated config: %s", err) + } + if err := encoder.Close(); err != nil { + return fmt.Errorf("error closing encoder: %s", err) } - encoder.Close() if err := os.WriteFile(localFilePath, buf.Bytes(), 0o644); err != nil { - return fmt.Errorf("Error overwriting local file %s: %s", configFile, err) + return fmt.Errorf("error overwriting local file %s: %s", configFile, err) } return nil } @@ -264,7 +266,7 @@ func updateComponentVersion(node *yaml.Node, path []string, version string) erro func retrieveBaseVersionConfig(baseVersion, repoRootDir string) error { gitHashOrTag, err := extractGitHashOrTag(baseVersion) if err != nil { - return fmt.Errorf("Error extracting git hash or tag from %q: %s", baseVersion, err) + return fmt.Errorf("error extracting git hash or tag from %q: %s", baseVersion, err) } for _, configFilePath := range []string{calicoConfig, enterpriseConfig} { @@ -277,7 +279,7 @@ func retrieveBaseVersionConfig(baseVersion, repoRootDir string) error { }).Debug("Replacing local file with downloaded file") if _, err := runCommand("curl", []string{"-L", "-o", localFilePath, url}, nil); err != nil { - return fmt.Errorf("Error downloading %s from %s: %s", configFilePath, url, err) + return fmt.Errorf("error downloading %s from %s: %s", configFilePath, url, err) } } return nil @@ -288,18 +290,18 @@ func retrieveBaseVersionConfig(baseVersion, repoRootDir string) error { func extractGitHashOrTag(baseVersion string) (string, error) { isReleaseVersion, err := isReleaseVersionFormat(baseVersion) if err != nil { - return "", fmt.Errorf("Error determining if version is a release: %s", err) + return "", fmt.Errorf("error determining if version is a release: %s", err) } if isReleaseVersion { return baseVersion, nil } gitHashRegex, err := regexp.Compile(`g([0-9a-f]{12})`) if err != nil { - return "", fmt.Errorf("Error compiling git hash regex: %s", err) + return "", fmt.Errorf("error compiling git hash regex: %s", err) } matches := gitHashRegex.FindStringSubmatch(baseVersion) if len(matches) > 1 { return matches[1], nil } - return "", fmt.Errorf("Error finding git hash in base version") + return "", fmt.Errorf("error finding git hash in base version") } diff --git a/hack/release-from/flags.go b/hack/release-from/flags.go index 8ad956b250..511a118a08 100644 --- a/hack/release-from/flags.go +++ b/hack/release-from/flags.go @@ -38,13 +38,9 @@ var gitRemoteFlag = &cli.StringFlag{ } var baseOperatorFlag = &cli.StringFlag{ - Name: "base-version", - Aliases: []string{"base"}, - Usage: "The version of the operator to base this new version from. " + - "It is expected in the format vX.Y.Z for releases and " + - "for hashrelease, either vX.Y.Z-n-g- (legacy) or " + - "vX.Y.Z--n-g- (new) " + - "where product-hashrelease-version is in the format vA.B.C--u-g.", + Name: "base-version", + Aliases: []string{"base"}, + Usage: "The version of the operator to use as the base for this new version.", Sources: cli.EnvVars("OPERATOR_BASE_VERSION"), Required: true, Action: func(ctx context.Context, c *cli.Command, value string) error { @@ -87,20 +83,15 @@ var versionFlag = &cli.StringFlag{ } var exceptCalicoFlag = &cli.StringSliceFlag{ - Name: "except-calico", - Usage: "A list of Calico images and the version to use for them. " + - "This should use the format based on the config/calico_versions.yaml file. " + - "e.g. --except-calico calico/cni:vX.Y.Z --except-calico csi-node-driver-registrar:vA.B.C-n-g", + Name: "except-calico", + Usage: "Calico image and version to update where the image name adheres with config/calico_versions.yaml file. Can be specified multiple times.", Sources: cli.EnvVars("OS_IMAGES_VERSIONS"), Action: validateOverrides, } var exceptEnterpriseFlag = &cli.StringSliceFlag{ Name: "except-calico-enterprise", - Aliases: []string{"except-enterprise", "except-calient"}, - Usage: "A list of Enterprise images and the versions to use for them. " + - "This should use the format based on the config/enterprise_versions.yaml file. " + - "e.g. --except-calico-enterprise linseed:vX.Y.Z --except-calico-enterprise security-event-webhooks-processor:vA.B.C-n-g", + Usage: "Enterprise image and version to update where image name adheres with config/enterprise_versions.yaml file. Can be specified multiple times.", Sources: cli.EnvVars("EE_IMAGES_VERSIONS"), Action: func(ctx context.Context, c *cli.Command, values []string) error { if len(values) == 0 && len(c.StringSlice("except-calico")) == 0 { @@ -125,7 +116,7 @@ var ( archFlag = &cli.StringSliceFlag{ Name: "architecture", Aliases: []string{"arch"}, - Usage: "The architecture to use for the release. Repeat for multiple architectures.", + Usage: "The architecture(s) for the release. Can be specified multiple times.", Sources: cli.EnvVars("ARCHS"), Value: archOptions, Action: func(ctx context.Context, c *cli.Command, values []string) error { @@ -140,17 +131,15 @@ var ( ) var registryFlag = &cli.StringFlag{ - Name: "registry", - Usage: "The registry to push the new operator to. " + - "This is only used when creating a new operator for hashreleases.", + Name: "registry", + Usage: "The registry to push the new operator to (ONLY for hashreleases operator).", Sources: cli.EnvVars("REGISTRY"), Value: quayRegistry, } var imageFlag = &cli.StringFlag{ - Name: "image", - Usage: "The image name to use for the new operator. " + - "This is only used when creating a new operator for hashreleases.", + Name: "image", + Usage: "The image name to use for the new operator (ONLY for hashreleases operator).", Sources: cli.EnvVars("IMAGE_NAME"), Value: defaultImageName, } diff --git a/hack/release-from/main.go b/hack/release-from/main.go index 720123ae9b..9ae580fdf6 100644 --- a/hack/release-from/main.go +++ b/hack/release-from/main.go @@ -47,9 +47,9 @@ func main() { } // check if git repo is dirty if version, err := gitVersion(); err != nil { - return ctx, fmt.Errorf("Error getting git version: %s", err) + return ctx, fmt.Errorf("error getting git version: %s", err) } else if strings.Contains(version, "dirty") { - return ctx, fmt.Errorf("Git repo is dirty, please commit changes before releasing") + return ctx, fmt.Errorf("git repo is dirty, please commit changes before releasing") } return ctx, nil }, From fa07d9c39e5581dfa25b7c91731dd9e542baea61 Mon Sep 17 00:00:00 2001 From: tuti Date: Thu, 23 Jan 2025 14:32:56 -0800 Subject: [PATCH 6/7] update README and add reset for hashrelease operator --- hack/release-from/README.md | 3 +++ hack/release-from/action.go | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/hack/release-from/README.md b/hack/release-from/README.md index dbdc5c88e3..7becb2b0e0 100644 --- a/hack/release-from/README.md +++ b/hack/release-from/README.md @@ -5,6 +5,9 @@ using a previously released operator version. The base operator version must reference either a tag or commit hash in `tigera/operator` +The new operator version will be built from the current codebase +with updates made to the image list based on the changes passed in. + ## Installation To install `release-from`, use the following command: diff --git a/hack/release-from/action.go b/hack/release-from/action.go index 1122570fde..e8075eaae3 100644 --- a/hack/release-from/action.go +++ b/hack/release-from/action.go @@ -104,6 +104,11 @@ func newOperator(dir, version, remote string, publish bool) error { // newHashreleaseOperator creates a new operator for a hashrelease. // if publish is true, it will also publish the operator to registry. func newHashreleaseOperator(dir, version, imageName, registry string, arches []string, publish bool) error { + defer func() { + if _, err := runCommandInDir(dir, "git", []string{"checkout", "config/"}, nil); err != nil { + logrus.WithError(err).Error("error reverting changes in config/") + } + } if err := buildHashreleaseOperator(dir, version, imageName, registry, arches); err != nil { return fmt.Errorf("error building operator: %s", err) } From 158dc66a7fdceb45275023d3e088d7944c1d02c2 Mon Sep 17 00:00:00 2001 From: tuti Date: Thu, 23 Jan 2025 16:45:52 -0800 Subject: [PATCH 7/7] fix CI --- hack/release-from/action.go | 8 ++-- ...projectcalico.org_felixconfigurations.yaml | 48 +++++++++++++++++++ 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/hack/release-from/action.go b/hack/release-from/action.go index e8075eaae3..7222eb3bd8 100644 --- a/hack/release-from/action.go +++ b/hack/release-from/action.go @@ -105,10 +105,10 @@ func newOperator(dir, version, remote string, publish bool) error { // if publish is true, it will also publish the operator to registry. func newHashreleaseOperator(dir, version, imageName, registry string, arches []string, publish bool) error { defer func() { - if _, err := runCommandInDir(dir, "git", []string{"checkout", "config/"}, nil); err != nil { - logrus.WithError(err).Error("error reverting changes in config/") - } - } + if _, err := runCommandInDir(dir, "git", []string{"checkout", "config/"}, nil); err != nil { + logrus.WithError(err).Error("error reverting changes in config/") + } + }() if err := buildHashreleaseOperator(dir, version, imageName, registry, arches); err != nil { return fmt.Errorf("error building operator: %s", err) } diff --git a/pkg/crds/calico/crd.projectcalico.org_felixconfigurations.yaml b/pkg/crds/calico/crd.projectcalico.org_felixconfigurations.yaml index 4903b4a764..04a2573888 100644 --- a/pkg/crds/calico/crd.projectcalico.org_felixconfigurations.yaml +++ b/pkg/crds/calico/crd.projectcalico.org_felixconfigurations.yaml @@ -227,6 +227,11 @@ spec: items: type: string type: array + bpfExportBufferSizeMB: + description: |- + BPFExportBufferSizeMB in BPF mode, controls the buffer size used for sending BPF events to felix. + [Default: 1] + type: integer bpfExtToServiceConnmark: description: |- BPFExtToServiceConnmark in BPF mode, controls a 32bit mark that is set on connections from an @@ -583,6 +588,43 @@ spec: - Enabled - Disabled type: string + flowLogsCollectorDebugTrace: + description: |- + When FlowLogsCollectorDebugTrace is set to true, enables the logs in the collector to be + printed in their entirety. + type: boolean + flowLogsEnableNetworkSets: + description: FlowLogsEnableNetworkSets enables Flow logs reporting + for GlobalNetworkSets. + type: boolean + flowLogsFileIncludeLabels: + description: FlowLogsFileIncludeLabels is used to configure if endpoint + labels are included in a Flow log entry written to file. + type: boolean + flowLogsFileIncludePolicies: + description: FlowLogsFileIncludePolicies is used to configure if policy + information are included in a Flow log entry written to file. + type: boolean + flowLogsFileIncludeService: + description: |- + FlowLogsFileIncludeService is used to configure if the destination service is included in a Flow log entry written to file. + The service information can only be included if the flow was explicitly determined to be directed at the service (e.g. + when the pre-DNAT destination corresponds to the service ClusterIP and port). + type: boolean + flowLogsFlushInterval: + description: FlowLogsFlushInterval configures the interval at which + Felix exports flow logs. + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + flowLogsGoldmaneServer: + description: FlowLogGoldmaneServer is the flow server endpoint to + which flow data should be published. + type: string + flowLogsMaxOriginalIPsIncluded: + description: FlowLogsMaxOriginalIPsIncluded specifies the number of + unique IP addresses (if relevant) that should be included in Flow + logs. + type: integer genericXDPEnabled: description: |- GenericXDPEnabled enables Generic XDP so network cards that don't support XDP offload or driver @@ -873,6 +915,12 @@ spec: routes, rules, and other kernel objects. [Default: 10s] pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ type: string + nfNetlinkBufSize: + description: |- + NfNetlinkBufSize controls the size of NFLOG messages that the kernel will try to send to Felix. NFLOG messages + are used to report flow verdicts from the kernel. Warning: currently increasing the value may cause errors + due to a bug in the netlink library. + type: string nftablesFilterAllowAction: description: |- NftablesFilterAllowAction controls the nftables action that Felix uses to represent the "allow" policy verdict