Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add esti tests for lakectl local with POSIX permissions #7962

Merged
merged 6 commits into from
Jul 7, 2024
Merged
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions esti/lakectl_local_test.go
Original file line number Diff line number Diff line change
@@ -255,6 +255,38 @@ func TestLakectlLocal_clone(t *testing.T) {
require.Contains(t, sanitizedResult, vars["PREFIX"]+"test2.txt")
require.NotContains(t, sanitizedResult, vars["PREFIX"]+"nodiff.txt")
})

t.Run("diff with posix permissions", func(t *testing.T) {
dataDir, err := os.MkdirTemp(tmpDir, "")
require.NoError(t, err)
vars["LOCAL_DIR"] = dataDir
vars["PREFIX"] = "posix"

lakectl := LakectlWithPosixPerms()
RunCmdAndVerifyContainsText(t, lakectl+" local clone lakefs://"+repoName+"/"+mainBranch+"/"+vars["PREFIX"]+" "+dataDir, false, "Successfully cloned lakefs://${REPO}/${REF}/${PREFIX}/ to ${LOCAL_DIR}.", vars)
localVerifyDirContents(t, dataDir, []string{})

// Add new files to path
localCreateTestData(t, vars, []string{
vars["PREFIX"] + uri.PathSeparator + "with-diff.txt",
vars["PREFIX"] + uri.PathSeparator + "no-diff.txt",
})

res := runCmd(t, lakectl+" local pull "+dataDir, false, false, vars)
require.Contains(t, res, "download with-diff.txt")
require.Contains(t, res, "download no-diff.txt")

// the following commit "initializes" the posix permissions for the remote repo
runCmd(t, lakectl+" local commit "+dataDir+" --allow-empty-message -m \" \"", false, false, vars)
Copy link
Contributor

@guy-har guy-har Jul 7, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: make it the commit message instead of a comment

Suggested change
// the following commit "initializes" the posix permissions for the remote repo
runCmd(t, lakectl+" local commit "+dataDir+" --allow-empty-message -m \" \"", false, false, vars)
commitMsg := "initialize the posix permissions for the remote repo"
runCmd(t, lakectl+" local commit "+dataDir+" -m " + commitMsg, false, false, vars)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TBH I prefer the fmt.Sprintf way, I noticed it's like that from previous tests as well

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think that fmt.Sprintf will make this much less readable.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Anyway, good ideas -
Updated.


Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe add another validation here to check no changes after commit

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done.

err = os.Chmod(filepath.Join(dataDir, "with-diff.txt"), 0755)
require.NoError(t, err)

sanitizedResult := runCmd(t, lakectl+" local status "+dataDir, false, false, vars)

require.Contains(t, sanitizedResult, "with-diff.txt")
require.NotContains(t, sanitizedResult, "no-diff.txt")
})
}

func TestLakectlLocal_pull(t *testing.T) {
9 changes: 9 additions & 0 deletions esti/lakectl_util.go
Original file line number Diff line number Diff line change
@@ -41,9 +41,14 @@ func lakectlLocation() string {
}

func LakectlWithParams(accessKeyID, secretAccessKey, endPointURL string) string {
return LakectlWithParamsWithPosixPerms(accessKeyID, secretAccessKey, endPointURL, false)
}

func LakectlWithParamsWithPosixPerms(accessKeyID, secretAccessKey, endPointURL string, withPosixPerms bool) string {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Adding this explicitly -
Rather keep this easy to delete (assuming it'll happen in the near future),
Than mess the whole code with changes (of adding a new param).

lakectlCmdline := "LAKECTL_CREDENTIALS_ACCESS_KEY_ID=" + accessKeyID +
" LAKECTL_CREDENTIALS_SECRET_ACCESS_KEY=" + secretAccessKey +
" LAKECTL_SERVER_ENDPOINT_URL=" + endPointURL +
" LAKECTL_EXPERIMENTAL_LOCAL_POSIX_PERMISSIONS_ENABLED=" + strconv.FormatBool(withPosixPerms) +
" " + lakectlLocation()

return lakectlCmdline
@@ -53,6 +58,10 @@ func Lakectl() string {
return LakectlWithParams(viper.GetString("access_key_id"), viper.GetString("secret_access_key"), viper.GetString("endpoint_url"))
}

func LakectlWithPosixPerms() string {
return LakectlWithParamsWithPosixPerms(viper.GetString("access_key_id"), viper.GetString("secret_access_key"), viper.GetString("endpoint_url"), true)
}

func runShellCommand(t *testing.T, command string, isTerminal bool) ([]byte, error) {
t.Helper()
t.Logf("Run shell command '%s'", command)
3 changes: 2 additions & 1 deletion pkg/local/sync.go
Original file line number Diff line number Diff line change
@@ -277,7 +277,8 @@ func (s *SyncManager) upload(ctx context.Context, rootPath string, remote *uri.U
if err := fileutil.VerifySafeFilename(source); err != nil {
return err
}
dest := strings.TrimPrefix(filepath.ToSlash(fmt.Sprintf("%s%s%s", remote.GetPath(), uri.PathSeparator, path)), uri.PathSeparator)
remotePath := strings.TrimRight(remote.GetPath(), uri.PathSeparator)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Regarding this change, see this PR.

dest := strings.TrimPrefix(filepath.ToSlash(fmt.Sprintf("%s%s%s", remotePath, uri.PathSeparator, path)), uri.PathSeparator)

f, err := os.Open(source)
if err != nil {