From 928fc14987a0e07ef3e5ca1fe2fc494fc8dc38e0 Mon Sep 17 00:00:00 2001 From: Sudhanshu Bawane Date: Thu, 1 Feb 2024 10:00:23 +0530 Subject: [PATCH 1/9] final Signed-off-by: Sudhanshu Bawane --- asset/boltdb_manager.go | 10 ++++++++++ asset/expander.go | 12 ++++++++++++ asset/expander_test.go | 30 ++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+) diff --git a/asset/boltdb_manager.go b/asset/boltdb_manager.go index eb9e008db6..a0b9bd850b 100644 --- a/asset/boltdb_manager.go +++ b/asset/boltdb_manager.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/spf13/viper" "os" "path/filepath" @@ -23,6 +24,7 @@ const ( // ExpandDuration is the name of the prometheus summary vec used to track // average latencies of asset expansion. ExpandDuration = "sensu_go_asset_expand_duration" + FlagCacheDir = "cache-dir" ) var ( @@ -241,6 +243,14 @@ func (b *boltDBAssetManager) expandWithDuration(tmpFile *os.File, asset *corev2. })) defer timer.ObserveDuration() + assetSHA := asset.Sha512 + CacheDir := viper.GetString(FlagCacheDir) + fullPath := filepath.Join(CacheDir, assetSHA) + + if err := CleanUp(fullPath); err != nil { //fix for git issue 5009 + logger.Printf("error cleaning up the SHA dir: %s", err) + } + assetPath = filepath.Join(b.localStorage, asset.Sha512) return assetPath, b.expander.Expand(tmpFile, assetPath) } diff --git a/asset/expander.go b/asset/expander.go index d13df1b1cd..79a1384838 100644 --- a/asset/expander.go +++ b/asset/expander.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "io" + "os" archiver "github.com/mholt/archiver/v3" @@ -90,3 +91,14 @@ func sniffType(f io.ReadSeeker) (filetype_types.Type, error) { return ft, nil } + +// Sudhanshu - CleanUp the SHA for the git issue 5009 fix. Making sure that in case of DOS when asset.db gets deleted it gets cleanUp so that asset can be re-downloded + +func CleanUp(fullPath string) error { + errorSHA := os.RemoveAll(fullPath) + if errorSHA != nil { + return errorSHA + } + return nil + +} diff --git a/asset/expander_test.go b/asset/expander_test.go index 6d028bdf90..44236c6b5e 100644 --- a/asset/expander_test.go +++ b/asset/expander_test.go @@ -113,3 +113,33 @@ func TestExpandInvalidArchive(t *testing.T) { t.Fail() } } + +// ---Test to check CleanUp +func TestCleanUp(t *testing.T) { + t.Parallel() + + // Create a temporary directory for testing + tmpDir := t.TempDir() + + // Define the SHA and file name + SHAName := "shaAsset.tar" + SHAFilePath := filepath.Join(tmpDir, SHAName) + + // Create a dummy file inside the temporary directory + SHAFile, err := os.Create(SHAFilePath) + if err != nil { + t.Fatalf("Failed to create dummy file: %v", err) + } + SHAFile.Close() + + // Call CleanUp with the SHA of the dummy file and the temporary directory + err = CleanUp(SHAFilePath) + if err != nil { + t.Errorf("CleanUp returned an error: %v", err) + } + + _, err = os.Stat(SHAFilePath) + if !os.IsNotExist(err) { + t.Errorf("CleanUp did not remove the dummy file as expected") + } +} From 151878211501e831a533aff8d607e5b5f37e07f2 Mon Sep 17 00:00:00 2001 From: SudhanshuBawane Date: Tue, 27 Feb 2024 22:49:42 +0530 Subject: [PATCH 2/9] final_fix Signed-off-by: SudhanshuBawane --- asset/boltdb_manager.go | 5 +++-- asset/expander.go | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/asset/boltdb_manager.go b/asset/boltdb_manager.go index a0b9bd850b..5ee0c84ddb 100644 --- a/asset/boltdb_manager.go +++ b/asset/boltdb_manager.go @@ -247,8 +247,9 @@ func (b *boltDBAssetManager) expandWithDuration(tmpFile *os.File, asset *corev2. CacheDir := viper.GetString(FlagCacheDir) fullPath := filepath.Join(CacheDir, assetSHA) - if err := CleanUp(fullPath); err != nil { //fix for git issue 5009 - logger.Printf("error cleaning up the SHA dir: %s", err) + if err := CleanUp(fullPath); err != nil { + logger.WithField("assetSHA path", fullPath).WithError(err). + Error("error cleaning up the assetSHA") } assetPath = filepath.Join(b.localStorage, asset.Sha512) diff --git a/asset/expander.go b/asset/expander.go index 79a1384838..7b3651a590 100644 --- a/asset/expander.go +++ b/asset/expander.go @@ -92,8 +92,7 @@ func sniffType(f io.ReadSeeker) (filetype_types.Type, error) { return ft, nil } -// Sudhanshu - CleanUp the SHA for the git issue 5009 fix. Making sure that in case of DOS when asset.db gets deleted it gets cleanUp so that asset can be re-downloded - +// cleanup of the assetSHA when cache dir gets force deleted func CleanUp(fullPath string) error { errorSHA := os.RemoveAll(fullPath) if errorSHA != nil { From 18f450c7ad738e591fa4c327cc6c50d5b9ce6b6c Mon Sep 17 00:00:00 2001 From: SudhanshuBawane Date: Thu, 7 Mar 2024 19:36:42 +0530 Subject: [PATCH 3/9] final Signed-off-by: SudhanshuBawane --- CHANGELOG-6.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG-6.md b/CHANGELOG-6.md index 5551cd6094..bbaab6542b 100644 --- a/CHANGELOG-6.md +++ b/CHANGELOG-6.md @@ -8,6 +8,14 @@ Versioning](http://semver.org/spec/v2.0.0.html). ## Unreleased +### 2024-02-01 + +### Changed +- The expandWithDuration in boltdb_manager to have a assetSHA dir cleanup. + +### Added +- Added the cleanup in expander to clean the assetSHA in case of asset cache getting deleted + ### Changed - Upgraded CI Go version to 1.21.3 - Upgraded jwt version to 4.4.3 From 001844fbd558ccaaeda97eefe1c8205d1ee61cb1 Mon Sep 17 00:00:00 2001 From: SudhanshuBawane Date: Wed, 13 Mar 2024 04:28:11 +0530 Subject: [PATCH 4/9] final Signed-off-by: SudhanshuBawane --- CHANGELOG-6.md | 7 ++----- asset/expander.go | 7 +------ 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/CHANGELOG-6.md b/CHANGELOG-6.md index bbaab6542b..275bab5cd9 100644 --- a/CHANGELOG-6.md +++ b/CHANGELOG-6.md @@ -10,11 +10,8 @@ Versioning](http://semver.org/spec/v2.0.0.html). ### 2024-02-01 -### Changed -- The expandWithDuration in boltdb_manager to have a assetSHA dir cleanup. - -### Added -- Added the cleanup in expander to clean the assetSHA in case of asset cache getting deleted +### Fixed +- Assets can now be re-installed when the asset.db file has been deleted. ### Changed - Upgraded CI Go version to 1.21.3 diff --git a/asset/expander.go b/asset/expander.go index 7b3651a590..ef59d7bc1c 100644 --- a/asset/expander.go +++ b/asset/expander.go @@ -94,10 +94,5 @@ func sniffType(f io.ReadSeeker) (filetype_types.Type, error) { // cleanup of the assetSHA when cache dir gets force deleted func CleanUp(fullPath string) error { - errorSHA := os.RemoveAll(fullPath) - if errorSHA != nil { - return errorSHA - } - return nil - + return os.RemoveAll(fullPath) } From 357ac9f01c801260e54eeb06d45009c1bfb4013e Mon Sep 17 00:00:00 2001 From: SudhanshuBawane Date: Tue, 19 Mar 2024 15:01:40 +0530 Subject: [PATCH 5/9] resolved Signed-off-by: SudhanshuBawane --- asset/boltdb_manager.go | 2 +- asset/boltdb_manager_test.go | 28 ++++++++++++++++++ asset/expander.go | 10 +++---- asset/expander_test.go | 56 ++++++++++++++++++------------------ 4 files changed, 61 insertions(+), 35 deletions(-) diff --git a/asset/boltdb_manager.go b/asset/boltdb_manager.go index 5ee0c84ddb..a3cc57e27f 100644 --- a/asset/boltdb_manager.go +++ b/asset/boltdb_manager.go @@ -247,7 +247,7 @@ func (b *boltDBAssetManager) expandWithDuration(tmpFile *os.File, asset *corev2. CacheDir := viper.GetString(FlagCacheDir) fullPath := filepath.Join(CacheDir, assetSHA) - if err := CleanUp(fullPath); err != nil { + if err := os.RemoveAll(fullPath); err != nil { logger.WithField("assetSHA path", fullPath).WithError(err). Error("error cleaning up the assetSHA") } diff --git a/asset/boltdb_manager_test.go b/asset/boltdb_manager_test.go index 6d494f006a..d41ca7a4ba 100644 --- a/asset/boltdb_manager_test.go +++ b/asset/boltdb_manager_test.go @@ -7,6 +7,7 @@ import ( "io" "io/ioutil" "os" + "path/filepath" "testing" bolt "go.etcd.io/bbolt" @@ -218,6 +219,33 @@ func TestFailedExpand(t *testing.T) { t.Log("expected error, got nil") t.Fail() } + + // Create a temporary directory for testing + tmpDir := t.TempDir() + + // Define the SHA and file name + SHAName := "shaAsset.tar" + SHAFilePath := filepath.Join(tmpDir, SHAName) + + // Create a dummy file inside the temporary directory + SHAFile, err := os.Create(SHAFilePath) + if err != nil { + t.Fatalf("Failed to create dummy file: %v", err) + } + SHAFile.Close() + + // Call CleanUp with the SHA of the dummy file and the temporary directory + err = os.RemoveAll(SHAFilePath) + if err != nil { + t.Errorf("CleanUp returned an error: %v", err) + t.Fail() + } + + _, err = os.Stat(SHAFilePath) + if !os.IsNotExist(err) { + t.Errorf("CleanUp did not remove the dummy file as expected") + t.Fail() + } } func TestSuccessfulGetAsset(t *testing.T) { diff --git a/asset/expander.go b/asset/expander.go index ef59d7bc1c..03929a6821 100644 --- a/asset/expander.go +++ b/asset/expander.go @@ -3,10 +3,8 @@ package asset import ( "errors" "fmt" - "io" - "os" - archiver "github.com/mholt/archiver/v3" + "io" filetype "gopkg.in/h2non/filetype.v1" filetype_types "gopkg.in/h2non/filetype.v1/types" @@ -93,6 +91,6 @@ func sniffType(f io.ReadSeeker) (filetype_types.Type, error) { } // cleanup of the assetSHA when cache dir gets force deleted -func CleanUp(fullPath string) error { - return os.RemoveAll(fullPath) -} +//func CleanUp(fullPath string) error { +// return os.RemoveAll(fullPath) +//} diff --git a/asset/expander_test.go b/asset/expander_test.go index 44236c6b5e..7ae10c7a7d 100644 --- a/asset/expander_test.go +++ b/asset/expander_test.go @@ -115,31 +115,31 @@ func TestExpandInvalidArchive(t *testing.T) { } // ---Test to check CleanUp -func TestCleanUp(t *testing.T) { - t.Parallel() - - // Create a temporary directory for testing - tmpDir := t.TempDir() - - // Define the SHA and file name - SHAName := "shaAsset.tar" - SHAFilePath := filepath.Join(tmpDir, SHAName) - - // Create a dummy file inside the temporary directory - SHAFile, err := os.Create(SHAFilePath) - if err != nil { - t.Fatalf("Failed to create dummy file: %v", err) - } - SHAFile.Close() - - // Call CleanUp with the SHA of the dummy file and the temporary directory - err = CleanUp(SHAFilePath) - if err != nil { - t.Errorf("CleanUp returned an error: %v", err) - } - - _, err = os.Stat(SHAFilePath) - if !os.IsNotExist(err) { - t.Errorf("CleanUp did not remove the dummy file as expected") - } -} +//func TestCleanUp(t *testing.T) { +// t.Parallel() +// +// // Create a temporary directory for testing +// tmpDir := t.TempDir() +// +// // Define the SHA and file name +// SHAName := "shaAsset.tar" +// SHAFilePath := filepath.Join(tmpDir, SHAName) +// +// // Create a dummy file inside the temporary directory +// SHAFile, err := os.Create(SHAFilePath) +// if err != nil { +// t.Fatalf("Failed to create dummy file: %v", err) +// } +// SHAFile.Close() +// +// // Call CleanUp with the SHA of the dummy file and the temporary directory +// err = CleanUp(SHAFilePath) +// if err != nil { +// t.Errorf("CleanUp returned an error: %v", err) +// } +// +// _, err = os.Stat(SHAFilePath) +// if !os.IsNotExist(err) { +// t.Errorf("CleanUp did not remove the dummy file as expected") +// } +//} From d2a0d01e4fc0cfd2e92b9cbd7da23cc0afe6975a Mon Sep 17 00:00:00 2001 From: SudhanshuBawane Date: Tue, 19 Mar 2024 15:03:37 +0530 Subject: [PATCH 6/9] resolved Signed-off-by: SudhanshuBawane --- asset/expander.go | 5 ----- asset/expander_test.go | 30 ------------------------------ 2 files changed, 35 deletions(-) diff --git a/asset/expander.go b/asset/expander.go index 03929a6821..1c1fd8fdff 100644 --- a/asset/expander.go +++ b/asset/expander.go @@ -89,8 +89,3 @@ func sniffType(f io.ReadSeeker) (filetype_types.Type, error) { return ft, nil } - -// cleanup of the assetSHA when cache dir gets force deleted -//func CleanUp(fullPath string) error { -// return os.RemoveAll(fullPath) -//} diff --git a/asset/expander_test.go b/asset/expander_test.go index 7ae10c7a7d..6d028bdf90 100644 --- a/asset/expander_test.go +++ b/asset/expander_test.go @@ -113,33 +113,3 @@ func TestExpandInvalidArchive(t *testing.T) { t.Fail() } } - -// ---Test to check CleanUp -//func TestCleanUp(t *testing.T) { -// t.Parallel() -// -// // Create a temporary directory for testing -// tmpDir := t.TempDir() -// -// // Define the SHA and file name -// SHAName := "shaAsset.tar" -// SHAFilePath := filepath.Join(tmpDir, SHAName) -// -// // Create a dummy file inside the temporary directory -// SHAFile, err := os.Create(SHAFilePath) -// if err != nil { -// t.Fatalf("Failed to create dummy file: %v", err) -// } -// SHAFile.Close() -// -// // Call CleanUp with the SHA of the dummy file and the temporary directory -// err = CleanUp(SHAFilePath) -// if err != nil { -// t.Errorf("CleanUp returned an error: %v", err) -// } -// -// _, err = os.Stat(SHAFilePath) -// if !os.IsNotExist(err) { -// t.Errorf("CleanUp did not remove the dummy file as expected") -// } -//} From ee21635063fc75ae449c809df49c04a7fcabb7b3 Mon Sep 17 00:00:00 2001 From: SudhanshuBawane Date: Thu, 21 Mar 2024 16:49:42 +0530 Subject: [PATCH 7/9] WIP Signed-off-by: SudhanshuBawane --- asset/boltdb_manager_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/asset/boltdb_manager_test.go b/asset/boltdb_manager_test.go index d41ca7a4ba..7e27199c38 100644 --- a/asset/boltdb_manager_test.go +++ b/asset/boltdb_manager_test.go @@ -221,6 +221,7 @@ func TestFailedExpand(t *testing.T) { } // Create a temporary directory for testing + //if -- dbName == "" true tmpDir := t.TempDir() // Define the SHA and file name From 5441dbb5bd7dd628d7ce682f413ee66b1571da46 Mon Sep 17 00:00:00 2001 From: SudhanshuBawane Date: Tue, 9 Apr 2024 10:36:00 +0530 Subject: [PATCH 8/9] comments resolved Signed-off-by: SudhanshuBawane --- asset/boltdb_manager_test.go | 47 +++++++++++++----------------------- 1 file changed, 17 insertions(+), 30 deletions(-) diff --git a/asset/boltdb_manager_test.go b/asset/boltdb_manager_test.go index 7e27199c38..9767e69e98 100644 --- a/asset/boltdb_manager_test.go +++ b/asset/boltdb_manager_test.go @@ -219,40 +219,12 @@ func TestFailedExpand(t *testing.T) { t.Log("expected error, got nil") t.Fail() } - - // Create a temporary directory for testing - //if -- dbName == "" true - tmpDir := t.TempDir() - - // Define the SHA and file name - SHAName := "shaAsset.tar" - SHAFilePath := filepath.Join(tmpDir, SHAName) - - // Create a dummy file inside the temporary directory - SHAFile, err := os.Create(SHAFilePath) - if err != nil { - t.Fatalf("Failed to create dummy file: %v", err) - } - SHAFile.Close() - - // Call CleanUp with the SHA of the dummy file and the temporary directory - err = os.RemoveAll(SHAFilePath) - if err != nil { - t.Errorf("CleanUp returned an error: %v", err) - t.Fail() - } - - _, err = os.Stat(SHAFilePath) - if !os.IsNotExist(err) { - t.Errorf("CleanUp did not remove the dummy file as expected") - t.Fail() - } } func TestSuccessfulGetAsset(t *testing.T) { t.Parallel() - - tmpFile, err := ioutil.TempFile(os.TempDir(), "asset_test_get_invalid_asset.db") + tmpDir := os.TempDir() + tmpFile, err := ioutil.TempFile(tmpDir, "asset_test_get_invalid_asset.db") if err != nil { t.Fatalf("unable to create test boltdb file: %v", err) } @@ -281,6 +253,21 @@ func TestSuccessfulGetAsset(t *testing.T) { URL: "path", } + tempAssetFile, err := os.CreateTemp(tmpDir, "asset.db") + if err != nil { + t.Logf("error creating the asset file as %v", err) + } + fi, err := tempAssetFile.Stat() + if fi.Size() == 0 { + t.Log("asset.db got created corruptly") + err = os.RemoveAll(filepath.Join(tmpDir, a.Sha512)) + if err != nil { + t.Logf("issue in deleting the assetSHA due to %v", err) + } else { + t.Log("assetSHA got deleted successfully and now asset will be properly downloaded.") + } + } + runtimeAsset, err := manager.Get(context.TODO(), a) if err != nil { t.Logf("expected no error, got: %v", err) From fb8b8a84c4a8f0569823e8b8e9a79b16abeaea1e Mon Sep 17 00:00:00 2001 From: SudhanshuBawane Date: Tue, 9 Apr 2024 10:42:40 +0530 Subject: [PATCH 9/9] comments resolved Signed-off-by: SudhanshuBawane --- asset/boltdb_manager_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/asset/boltdb_manager_test.go b/asset/boltdb_manager_test.go index 9767e69e98..9ea6c301b8 100644 --- a/asset/boltdb_manager_test.go +++ b/asset/boltdb_manager_test.go @@ -257,7 +257,7 @@ func TestSuccessfulGetAsset(t *testing.T) { if err != nil { t.Logf("error creating the asset file as %v", err) } - fi, err := tempAssetFile.Stat() + fi, _ := tempAssetFile.Stat() if fi.Size() == 0 { t.Log("asset.db got created corruptly") err = os.RemoveAll(filepath.Join(tmpDir, a.Sha512))