-
Notifications
You must be signed in to change notification settings - Fork 43
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add support for incremental stamping #175
Changes from 2 commits
9fc8ad8
8edf71c
b825973
ae3b54f
df89ffc
45169cd
5f6cb0e
50872a1
c423c37
958f48a
7379e81
2e73b01
8d6d022
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,20 @@ | ||
# Copyright 2024 Adobe. All rights reserved. | ||
# This file is licensed to you under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. You may obtain a copy | ||
# of the License at http://www.apache.org/licenses/LICENSE-2.0 | ||
|
||
# Unless required by applicable law or agreed to in writing, software distributed under | ||
# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS | ||
# OF ANY KIND, either express or implied. See the License for the specific language | ||
# governing permissions and limitations under the License. | ||
|
||
load("@io_bazel_rules_go//go:def.bzl", "go_library") | ||
|
||
licenses(["notice"]) # Apache 2.0 | ||
|
||
go_library( | ||
name = "go_default_library", | ||
srcs = ["digester.go"], | ||
importpath = "github.com/adobe/rules_gitops/gitops/digester", | ||
visibility = ["//visibility:public"], | ||
) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,74 @@ | ||
/* | ||
Copyright 2024 Adobe. All rights reserved. | ||
This file is licensed to you under the Apache License, Version 2.0 (the "License"); | ||
you may not use this file except in compliance with the License. You may obtain a copy | ||
of the License at http://www.apache.org/licenses/LICENSE-2.0 | ||
|
||
Unless required by applicable law or agreed to in writing, software distributed under | ||
the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS | ||
OF ANY KIND, either express or implied. See the License for the specific language | ||
governing permissions and limitations under the License. | ||
*/ | ||
package digester | ||
|
||
import ( | ||
"crypto/sha256" | ||
"encoding/hex" | ||
"errors" | ||
"io" | ||
"log" | ||
"os" | ||
) | ||
|
||
// CalculateDigest calculates the SHA256 digest of a file specified by the given path | ||
func CalculateDigest(path string) string { | ||
if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { | ||
return "" | ||
} | ||
|
||
fi, err := os.Open(path) | ||
if err != nil { | ||
log.Fatal(err) | ||
} | ||
defer fi.Close() | ||
|
||
h := sha256.New() | ||
if _, err := io.Copy(h, fi); err != nil { | ||
log.Fatal(err) | ||
} | ||
|
||
return hex.EncodeToString(h.Sum(nil)) | ||
} | ||
|
||
// GetDigest retrieves the digest of a file from a file with the same name but with a ".digest" extension | ||
func GetDigest(path string) string { | ||
digestPath := path + ".digest" | ||
|
||
if _, err := os.Stat(digestPath); errors.Is(err, os.ErrNotExist) { | ||
return "" | ||
} | ||
|
||
digest, err := os.ReadFile(digestPath) | ||
if err != nil { | ||
log.Fatal(err) | ||
} | ||
|
||
return string(digest) | ||
} | ||
|
||
// VerifyDigest verifies the integrity of a file by comparing its calculated digest with the stored digest | ||
func VerifyDigest(path string) bool { | ||
return CalculateDigest(path) == GetDigest(path) | ||
} | ||
|
||
// SaveDigest calculates the digest of a file at the given path and saves it to a file with the same name but with a ".digest" extension. | ||
func SaveDigest(path string) { | ||
digest := CalculateDigest(path) | ||
|
||
digestPath := path + ".digest" | ||
|
||
err := os.WriteFile(digestPath, []byte(digest), 0666) | ||
if err != nil { | ||
log.Fatal(err) | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -18,6 +18,7 @@ import ( | |
"os" | ||
oe "os/exec" | ||
"path/filepath" | ||
"strings" | ||
|
||
"github.com/adobe/rules_gitops/gitops/exec" | ||
) | ||
|
@@ -103,6 +104,34 @@ func (r *Repo) Commit(message, gitopsPath string) bool { | |
return true | ||
} | ||
|
||
// RemoveDiff removes the changes made to a specific file in the repository | ||
func (r *Repo) RemoveDiff(fileName string) { | ||
exec.Mustex(r.Dir, "git", "checkout", "--", fileName) | ||
} | ||
|
||
// split by newline and ignore empty strings | ||
func SplitFunc(c rune) bool { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This function doesn't seem to belong to the public interface of |
||
return c == '\n' | ||
} | ||
|
||
// GetChangedFiles returns a list of files that have been changed in the repository | ||
func (r *Repo) GetChangedFiles() []string { | ||
files, err := exec.Ex(r.Dir, "git", "diff", "--name-only") | ||
if err != nil { | ||
log.Fatalf("ERROR: %s", err) | ||
} | ||
return strings.FieldsFunc(files, SplitFunc) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'd prefer the use of the var files []string
sc := bufio.NewScanner(strings.NewReader(s))
for sc.Scan() {
files = append(files, sc.Text())
}
return lines |
||
} | ||
|
||
// GetCommitSha returns the SHA of the current commit | ||
func (r *Repo) GetCommitSha() string { | ||
commit, err := exec.Ex(r.Dir, "git", "rev-parse", "HEAD") | ||
if err != nil { | ||
log.Fatalf("ERROR: %s", err) | ||
} | ||
return strings.TrimSpace(commit) | ||
} | ||
|
||
// IsClean returns true if there is no local changes (nothing to commit) | ||
func (r *Repo) IsClean() bool { | ||
cmd := oe.Command("git", "status", "--porcelain") | ||
|
Original file line number | Diff line number | Diff line change | ||||||||||||||
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
@@ -25,11 +25,13 @@ import ( | |||||||||||||||
"github.com/adobe/rules_gitops/gitops/analysis" | ||||||||||||||||
"github.com/adobe/rules_gitops/gitops/bazel" | ||||||||||||||||
"github.com/adobe/rules_gitops/gitops/commitmsg" | ||||||||||||||||
"github.com/adobe/rules_gitops/gitops/digester" | ||||||||||||||||
"github.com/adobe/rules_gitops/gitops/exec" | ||||||||||||||||
"github.com/adobe/rules_gitops/gitops/git" | ||||||||||||||||
"github.com/adobe/rules_gitops/gitops/git/bitbucket" | ||||||||||||||||
"github.com/adobe/rules_gitops/gitops/git/github" | ||||||||||||||||
"github.com/adobe/rules_gitops/gitops/git/gitlab" | ||||||||||||||||
"github.com/adobe/rules_gitops/templating/fasttemplate" | ||||||||||||||||
|
||||||||||||||||
proto "github.com/golang/protobuf/proto" | ||||||||||||||||
) | ||||||||||||||||
|
@@ -71,6 +73,7 @@ var ( | |||||||||||||||
gitopsRuleName SliceFlags | ||||||||||||||||
gitopsRuleAttr SliceFlags | ||||||||||||||||
dryRun = flag.Bool("dry_run", false, "Do not create PRs, just print what would be done") | ||||||||||||||||
stamp = flag.Bool("stamp", false, "Stamp results of gitops targets with volatile information") | ||||||||||||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
Historically we kept |
||||||||||||||||
) | ||||||||||||||||
|
||||||||||||||||
func init() { | ||||||||||||||||
|
@@ -100,6 +103,40 @@ func bazelQuery(query string) *analysis.CqueryResult { | |||||||||||||||
return qr | ||||||||||||||||
} | ||||||||||||||||
|
||||||||||||||||
func getContext(workdir *git.Repo, branchName string) map[string]interface{} { | ||||||||||||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||||||||||||
commitSha := workdir.GetCommitSha() | ||||||||||||||||
|
||||||||||||||||
utcDate, err := exec.Ex("", "date", "-u") | ||||||||||||||||
if err != nil { | ||||||||||||||||
log.Fatal(err) | ||||||||||||||||
} | ||||||||||||||||
utcDate = strings.TrimSpace(utcDate) | ||||||||||||||||
|
||||||||||||||||
ctx := map[string]interface{}{ | ||||||||||||||||
"GIT_REVISION": commitSha, | ||||||||||||||||
"UTC_DATE": utcDate, | ||||||||||||||||
"GIT_BRANCH": branchName, | ||||||||||||||||
} | ||||||||||||||||
|
||||||||||||||||
return ctx | ||||||||||||||||
} | ||||||||||||||||
|
||||||||||||||||
func stampFile(fullPath string, workdir *git.Repo, branchName string) { | ||||||||||||||||
template, err := os.ReadFile(fullPath) | ||||||||||||||||
if err != nil { | ||||||||||||||||
log.Fatal(err) | ||||||||||||||||
} | ||||||||||||||||
|
||||||||||||||||
ctx := getContext(workdir, branchName) | ||||||||||||||||
|
||||||||||||||||
stampedTemplate := fasttemplate.ExecuteString(string(template), "{{", "}}", ctx) | ||||||||||||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||||||||||||
|
||||||||||||||||
err = os.WriteFile(fullPath, []byte(stampedTemplate), 0666) | ||||||||||||||||
if err != nil { | ||||||||||||||||
log.Fatal(err) | ||||||||||||||||
} | ||||||||||||||||
} | ||||||||||||||||
|
||||||||||||||||
func main() { | ||||||||||||||||
flag.Parse() | ||||||||||||||||
if *workspace != "" { | ||||||||||||||||
|
@@ -185,6 +222,18 @@ func main() { | |||||||||||||||
bin := bazel.TargetToExecutable(target) | ||||||||||||||||
exec.Mustex("", bin, "--nopush", "--nobazel", "--deployment_root", gitopsdir) | ||||||||||||||||
} | ||||||||||||||||
if *stamp { | ||||||||||||||||
changedFiles := workdir.GetChangedFiles() | ||||||||||||||||
for _, filePath := range changedFiles { | ||||||||||||||||
fullPath := gitopsdir + "/" + filePath | ||||||||||||||||
if digester.VerifyDigest(fullPath) { | ||||||||||||||||
workdir.RemoveDiff(fullPath) | ||||||||||||||||
} else { | ||||||||||||||||
digester.SaveDigest(fullPath) | ||||||||||||||||
stampFile(fullPath, workdir, *branchName) | ||||||||||||||||
} | ||||||||||||||||
} | ||||||||||||||||
} | ||||||||||||||||
if workdir.Commit(fmt.Sprintf("GitOps for release branch %s from %s commit %s\n%s", *releaseBranch, *branchName, *gitCommit, commitmsg.Generate(targets)), *gitopsPath) { | ||||||||||||||||
log.Println("branch", branch, "has changes, push is required") | ||||||||||||||||
updatedGitopsTargets = append(updatedGitopsTargets, targets...) | ||||||||||||||||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -540,16 +540,24 @@ def _kubectl_impl(ctx): | |
transitive_runfiles += [exe[DefaultInfo].default_runfiles for exe in trans_img_pushes] | ||
|
||
namespace = ctx.attr.namespace | ||
|
||
variables = "--variable=NAMESPACE={namespace}".format( | ||
namespace = namespace, | ||
) | ||
variables += " --variable=GIT_REVISION=\"$(git rev-parse HEAD)\"" | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We should not do stamping of stamps in
|
||
variables += " --variable=UTC_DATE=\"$(date -u)\"" | ||
variables += " --variable=GIT_BRANCH=\"$(git rev-parse --abbrev-ref HEAD)\"" | ||
|
||
for inattr in ctx.attr.srcs: | ||
for infile in inattr.files.to_list(): | ||
statements += "{template_engine} --template={infile} --variable=NAMESPACE={namespace} --stamp_info_file={info_file} | kubectl --cluster=\"{cluster}\" --user=\"{user}\" {kubectl_command} -f -\n".format( | ||
statements += "{template_engine} --template={infile} {variables} --stamp_info_file={info_file} | kubectl --cluster=\"{cluster}\" --user=\"{user}\" {kubectl_command} -f -\n".format( | ||
infile = infile.short_path, | ||
cluster = cluster_arg, | ||
user = user_arg, | ||
kubectl_command = kubectl_command_arg, | ||
template_engine = "${RUNFILES}/%s" % _get_runfile_path(ctx, ctx.executable._template_engine), | ||
namespace = namespace, | ||
info_file = ctx.file._info_file.short_path, | ||
variables = variables, | ||
) | ||
|
||
ctx.actions.expand_template( | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Align with the terminology used by Git. For example: